godzil/crunchy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
Manoël Trapier 2020-04-13 20:25:37 +01:00 committed by GitHub
commit f086b3593e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 280 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
package.json
View File

@ -15,41 +15,41 @@
"engines": {
"node": ">=5.0"
},
"version": "1.5.0",
"version": "1.5.1",
"bin": {
"crunchy": "./bin/crunchy",
"crunchy.sh": "./bin/crunchy.sh"
},
"dependencies": {
"big-integer": "^1.6.44",
"bluebird": "^3.5.5",
"big-integer": "^1.6.48",
"bluebird": "^3.7.2",
"brotli": "^1.3.2",
"cheerio": "^0.22.0",
"cloudscraper": "^4.1.2",
"commander": "^2.20.0",
"fs-extra": "^8.1.0",
"mkdirp": "^0.5.0",
"cloudscraper": "^4.6.0",
"commander": "^5.0.0",
"fs-extra": "^9.0.0",
"mkdirp": "^1.0.4",
"pjson": "^1.0.9",
"request": "^2.88.0",
"request-promise": "^4.2.4",
"request": "^2.88.2",
"request-promise": "^4.2.5",
"tough-cookie-file-store": "^1.2.0",
"uuid": "^3.3.2",
"xml2js": "^0.4.5"
"uuid": "^7.0.3",
"xml2js": "^0.4.23"
},
"devDependencies": {
"@types/bluebird": "^3.5.27",
"@types/cheerio": "^0.22.12",
"@types/fs-extra": "^8.0.0",
"@types/mkdirp": "^0.5.2",
"@types/node": "^12.6.8",
"@types/request": "^2.48.2",
"@types/request-promise": "^4.1.44",
"@types/uuid": "^3.4.5",
"@types/xml2js": "^0.4.4",
"npm-check": "^5.9.0",
"@types/bluebird": "^3.5.30",
"@types/cheerio": "^0.22.17",
"@types/fs-extra": "^8.1.0",
"@types/mkdirp": "^1.0.0",
"@types/node": "^13.11.1",
"@types/request": "^2.48.4",
"@types/request-promise": "^4.1.46",
"@types/uuid": "^7.0.2",
"@types/xml2js": "^0.4.5",
"npm-check": "^5.9.2",
"tsconfig-lint": "^0.12.0",
"tslint": "^5.18.0",
"typescript": "^3.5.3"
"tslint": "^6.1.1",
"typescript": "^3.8.3"
},
"scripts": {
"prepublishOnly": "npm run build",

46
src/batch.ts
View File

@ -200,7 +200,7 @@ function split(value: string): string[]
return pieces;
}
function get_min_filter(filter: string): number
function get_min_filter(filter: string): IEpisodeNumber
{
if (filter !== undefined)
{
@ -214,13 +214,41 @@ function get_min_filter(filter: string): number
if (tok[0] !== '')
{
return parseInt(tok[0], 10);
/* If first item is not empty, ie '10-20' */
if (tok[0].includes('e'))
{
/* include a e so we probably have something like 5e10
aka season 5 episode 10
*/
const tok2 = tok[0].split('else');
if (tok2.length > 2)
{
log.error('Invalid episode filter \'' + filter + '\'');
process.exit(-1);
}
if (tok[0] !== '')
{
/* So season is properly filled */
return {season: parseInt(tok2[0], 10), episode: parseInt(tok2[1], 10)};
}
else
{
/* we have 'e10' */
return {season: 0, episode: parseInt(tok2[1], 10)};
}
}
return 0;
else
{
return {season: 0, episode: parseInt(tok[0], 10)};
}
}
}
/* First item is empty, ie '-20' */
return {season: 0, episode: 0};
}
function get_max_filter(filter: string): number
function get_max_filter(filter: string): IEpisodeNumber
{
if (filter !== undefined)
{
@ -235,15 +263,15 @@ function get_max_filter(filter: string): number
if ((tok.length > 1) && (tok[1] !== ''))
{
/* We have a max value */
return parseInt(tok[1], 10);
return {season: +Infinity, episode: parseInt(tok[1], 10)};
}
else if ((tok.length === 1) && (tok[0] !== ''))
{
/* A single episode has been requested */
return parseInt(tok[0], 10);
return {season: +Infinity, episode: parseInt(tok[0], 10) + 1};
}
}
return +Infinity;
return {season: +Infinity, episode: +Infinity};
}
/**
@ -290,7 +318,7 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
episode_min: get_min_filter(config.episodes), episode_max: get_max_filter(config.episodes)};
}
return {address: '', retry: 0, episode_min: 0, episode_max: 0};
return {address: '', retry: 0, episode_min: {season: 0, episode: 0}, episode_max: {season: 0, episode: 0}};
}));
}
@ -366,6 +394,6 @@ function parse(args: string[]): IConfigLine
.option('--verbose', 'Make tool verbose')
.option('--debug', 'Create a debug file. Use only if requested!')
.option('--rebuildcrp', 'Rebuild the crpersistant file.')
.option('--retry <i>', 'Number or time to retry fetching an episode.', 5)
.option('--retry <i>', 'Number or time to retry fetching an episode.', '5')
.parse(args);
}

21
src/config.ts
View File

@ -45,6 +45,27 @@ export function save(config: IConfig)
tmp.args = undefined;
tmp.commands = undefined;
tmp._allowUnknownOption = undefined;
tmp.parent = undefined;
tmp._scriptPath = undefined;
tmp._optionValues = undefined;
tmp._storeOptionsAsProperties = undefined;
tmp._passCommandToAction = undefined;
tmp._actionResults = undefined;
tmp._actionHandler = undefined;
tmp._executableHandler = undefined;
tmp._executableFile = undefined;
tmp._defaultCommandName = undefined;
tmp._exitCallback = undefined;
tmp._alias = undefined;
tmp._noHelp = undefined;
tmp._helpFlags = undefined;
tmp._helpDescription = undefined;
tmp._helpShortFlag = undefined;
tmp._helpLongFlag = undefined;
tmp._hasImplicitHelpCommand = undefined;
tmp._helpCommandName = undefined;
tmp._helpCommandnameAndArgs = undefined;
tmp._helpCommandDescription = undefined;
// Things we don't want to save
tmp.cache = undefined;

34
src/episode.ts
View File

@ -66,13 +66,15 @@ function fileExist(path: string)
function sanitiseFileName(str: string)
{
return str.replace(/[\/':\?\*"<>\\\.\|]/g, '_');
const sanitized = str.replace(/[\/':\?\*"<>\\\.\|]/g, '_');
return sanitized.replace(/{DIR_SEPARATOR}/g, '/');
}
/**
* Downloads the subtitle and video.
*/
function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void)
function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error | string, ign: boolean) => void)
{
const serieFolder = sanitiseFileName(config.series || page.series);
@ -109,14 +111,9 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
return done(null, true);
}
mkdirp(path.dirname(filePath), (errM: Error) =>
const ret = mkdirp(path.dirname(filePath));
if (ret)
{
if (errM)
{
log.dispEpisode(fileName, 'Error...', true);
return done(errM, false);
}
log.dispEpisode(fileName, 'Fetching...', false);
downloadSubtitle(config, player, filePath, (errDS) =>
{
@ -164,13 +161,18 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
done(null, true);
}
});
});
}
else
{
log.dispEpisode(fileName, 'Error creating folder \'" + filePath + "\'...', true);
return done('Cannot create folder', false);
}
}
/**
* Saves the subtitles to disk.
*/
function downloadSubtitle(config: IConfig, player: IEpisodePlayer, filePath: string, done: (err?: Error) => void)
function downloadSubtitle(config: IConfig, player: IEpisodePlayer, filePath: string, done: (err?: Error | string) => void)
{
const enc = player.subtitle;
@ -336,15 +338,15 @@ function scrapePlayer(config: IConfig, address: string, id: number, done: (err:
return done(new Error('Invalid address.'));
}
my_request.post(config, {
form: {
const postForm = {
current_page: address,
video_format: config.video_format,
video_quality: config.video_quality,
media_id: id
},
url: url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id,
}, (err, result) =>
};
my_request.post(config, url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id, postForm,
(err, result) =>
{
if (err)
{

2
src/interface/AuthError.d.ts vendored
View File

@ -1,3 +1,5 @@
interface IAuthError extends Error {
name: string;
message: string;
authError: boolean;
}

4
src/interface/IConfigTask.d.ts vendored
View File

@ -1,6 +1,6 @@
interface IConfigTask {
address: string;
retry: number;
episode_min: number;
episode_max: number;
episode_min: IEpisodeNumber;
episode_max: IEpisodeNumber;
}

4
src/interface/IEpisodeNumber.d.ts vendored Normal file
View File

@ -0,0 +1,4 @@
interface IEpisodeNumber {
season: number,
episode: number
}

209
src/my_request.ts
View File

@ -22,26 +22,10 @@ let isPremium = false;
let j: request.CookieJar;
const defaultHeaders =
{
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
'Connection': 'keep-alive',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Referer': 'https://www.crunchyroll.com/login',
'Cache-Control': 'private',
'Accept-Language': 'en-US,en;q=0.9'
};
const defaultOptions =
{
followAllRedirects: true,
decodeEmails: true,
challengesToSolve: 3,
gzip: true,
};
// tslint:disable-next-line:no-var-requires
const cloudscraper = require('cloudscraper').defaults(defaultOptions);
import cloudscraper = require('cloudscraper');
let currentOptions: any;
let optionsSet = false;
function AuthError(msg: string): IAuthError
{
@ -98,19 +82,14 @@ function APIlogin(config: IConfig, sessionId: string, user: string, pass: string
});
}
function checkIfUserIsAuth(config: IConfig, done: (err: Error) => void): void
function checkIfUserIsAuth(config: IConfig, done: (err: any) => void): void
{
if (j === undefined)
{
loadCookies(config);
}
/**
* The main page give us some information about the user
*/
const url = 'http://www.crunchyroll.com/';
cloudscraper.get({gzip: true, uri: url, jar: j}, (err: Error, rep: string, body: string) =>
cloudscraper.get(url, getOptions(config, null), (err: any, rep: Response, body: string) =>
{
if (err)
{
@ -204,24 +183,14 @@ export function eatCookies(config: IConfig)
export function getUserAgent(): string
{
return defaultHeaders['User-Agent'];
return currentOptions.headers['User-Agent'];
}
/**
* Performs a GET request for the resource.
*/
export function get(config: IConfig, options: string|any, done: (err: any, result?: string) => void)
export function get(config: IConfig, url: string, done: (err: any, result?: string) => void)
{
if (j === undefined)
{
loadCookies(config);
}
if (config.userAgent)
{
defaultHeaders['User-Agent'] = config.userAgent;
}
authenticate(config, (err) =>
{
if (err)
@ -229,7 +198,7 @@ export function get(config: IConfig, options: string|any, done: (err: any, resul
return done(err);
}
cloudscraper.get(modify(options), (error: any, response: any, body: any) =>
cloudscraper.get(url, getOptions(config, null), (error: any, response: any, body: any) =>
{
if (error) return done(error);
@ -241,18 +210,8 @@ export function get(config: IConfig, options: string|any, done: (err: any, resul
/**
* Performs a POST request for the resource.
*/
export function post(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void)
export function post(config: IConfig, url: string, form: any, done: (err: any, result?: string) => void)
{
if (j === undefined)
{
loadCookies(config);
}
if (config.userAgent)
{
defaultHeaders['User-Agent'] = config.userAgent;
}
authenticate(config, (err) =>
{
if (err)
@ -260,7 +219,7 @@ export function post(config: IConfig, options: request.Options, done: (err: Erro
return done(err);
}
cloudscraper.post(modify(options), (error: Error, response: any, body: any) =>
cloudscraper.post(url, getOptions(config, form), (error: Error, response: any, body: any) =>
{
if (error)
{
@ -271,35 +230,32 @@ export function post(config: IConfig, options: request.Options, done: (err: Erro
});
}
/**
* Authenticates using the configured pass and user.
*/
function authenticate(config: IConfig, done: (err: Error) => void)
function authUsingCookies(config: IConfig, done: (err: any) => void)
{
if (isAuthenticated)
{
return done(null);
}
j.setCookie(request.cookie('session_id=' + config.crSessionId + '; Domain=crunchyroll.com; HttpOnly; hostOnly=false;'),
CR_COOKIE_DOMAIN);
/* First of all, check if the user is not already logged via the cookies */
checkIfUserIsAuth(config, (errCheckAuth) =>
checkIfUserIsAuth(config, (errCheckAuth2) =>
{
if (isAuthenticated)
{
return done(null);
}
else
{
return done(errCheckAuth2);
}
});
}
/* So if we are here now, that mean we are not authenticated so do as usual */
if ((!config.logUsingApi && !config.logUsingCookie) && (!config.pass || !config.user))
function authUsingApi(config: IConfig, done: (err: any) => void)
{
if (!config.pass || !config.user)
{
log.error('You need to give login/password to use Crunchy');
process.exit(-1);
}
log.info('Seems we are not currently logged. Let\'s login!');
if (config.logUsingApi)
{
if (config.crDeviceId === undefined)
{
config.crDeviceId = uuid.v4();
@ -335,34 +291,19 @@ function authenticate(config: IConfig, done: (err: Error) => void)
{
return done(AuthError(errInChk.message));
});
}
else if (config.logUsingCookie)
{
j.setCookie(request.cookie('session_id=' + config.crSessionId + '; Domain=crunchyroll.com; HttpOnly; hostOnly=false;'),
CR_COOKIE_DOMAIN);
}
checkIfUserIsAuth(config, (errCheckAuth2) =>
function authUsingForm(config: IConfig, done: (err: any) => void)
{
/* So if we are here now, that mean we are not authenticated so do as usual */
if (!config.pass || !config.user)
{
if (isAuthenticated)
{
return done(null);
log.error('You need to give login/password to use Crunchy');
process.exit(-1);
}
else
{
return done(errCheckAuth2);
}
});
}
else
{
/* First get https://www.crunchyroll.com/login to get the login token */
const options =
{
// jar: j,
uri: 'https://www.crunchyroll.com/login'
};
cloudscraper.get(options, (err: Error, rep: string, body: string) =>
cloudscraper.get('https://www.crunchyroll.com/login', getOptions(config, null), (err: any, rep: Response, body: string) =>
{
if (err) return done(err);
@ -376,20 +317,15 @@ function authenticate(config: IConfig, done: (err: Error) => void)
}
/* Now call the page again with the token and credentials */
const options =
{
form:
const paramForm =
{
'login_form[name]': config.user,
'login_form[password]': config.pass,
'login_form[redirect_url]': '/',
'login_form[_token]': token
},
// jar: j,
url: 'https://www.crunchyroll.com/login'
};
cloudscraper.post(options, (err: Error, rep: string, body: string) =>
cloudscraper.post('https://www.crunchyroll.com/login', getOptions(config, paramForm), (err: any, rep: Response, body: string) =>
{
if (err)
{
@ -410,22 +346,77 @@ function authenticate(config: IConfig, done: (err: Error) => void)
});
});
});
}
/**
* Authenticates using the configured pass and user.
*/
function authenticate(config: IConfig, done: (err: any) => void)
{
if (isAuthenticated)
{
return done(null);
}
/* First of all, check if the user is not already logged via the cookies */
checkIfUserIsAuth(config, (errCheckAuth) =>
{
if (isAuthenticated)
{
return done(null);
}
log.info('Seems we are not currently logged. Let\'s login!');
if (config.logUsingApi)
{
return authUsingApi(config, done);
}
else if (config.logUsingCookie)
{
return authUsingCookies(config, done);
}
else
{
return authUsingForm(config, done);
}
});
}
/**
* Modifies the options to use the authenticated cookie jar.
*/
function modify(options: string|any): any
function getOptions(config: IConfig, form: any)
{
if (typeof options !== 'string')
if (!optionsSet)
{
options.jar = j;
return options;
currentOptions = {};
currentOptions.headers = {};
if (config.userAgent)
{
currentOptions.headers['User-Agent'] = config.userAgent;
}
return {
jar: j,
url: options.toString(),
};
else
{
currentOptions.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0';
}
if (j === undefined)
{
loadCookies(config);
}
currentOptions.decodeEmails = true;
currentOptions.jar = j;
optionsSet = true;
}
currentOptions.form = {};
if (form !== null)
{
currentOptions.form = form;
}
return currentOptions;
}

5
src/series.ts
View File

@ -144,9 +144,10 @@ function download(cache: {[address: string]: number}, config: IConfig,
done: (err: any, ign: boolean) => void)
{
const episodeNumber = parseInt(item.episode, 10);
const seasonNumber = item.volume;
if ( (episodeNumber < task.episode_min) ||
(episodeNumber > task.episode_max) )
if ( (episodeNumber < task.episode_min.episode) ||
(episodeNumber > task.episode_max.episode) )
{
return done(null, false);
}