Compare commits
98 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de3f9a1e9e | ||
|
|
f26ea70ef8 | ||
|
|
baf15dc1b4 | ||
|
|
2c58e5e4ba | ||
|
|
c8f33e947d | ||
|
|
fc119acb1c | ||
|
|
2db34c3ed8 | ||
|
|
64200a1da9 | ||
|
|
8655874097 | ||
|
|
99ba051b7f | ||
|
|
d692199d07 | ||
|
|
e058b8e699 | ||
|
|
b3a96cd0e7 | ||
|
|
f25a62234c | ||
|
|
e8a7856e44 | ||
|
|
f086b3593e | ||
|
|
d2bc7e41b2 | ||
|
|
a7bc34df0d | ||
|
|
b2ecd05586 | ||
|
|
8b9f9a5e1c | ||
|
|
376ff09632 | ||
|
|
7926b2fd9a | ||
|
|
3f2a920e1e | ||
|
|
fa4c68c239 | ||
|
|
1555229635 | ||
|
|
7c085caaa0 | ||
|
|
cc0bf4dfb1 | ||
|
|
00857ba46f | ||
|
|
b77a35e0e9 | ||
|
|
ca59e3b2fd | ||
|
|
95c0c4d6d3 | ||
|
|
0d2d36251a | ||
|
|
48a58ffca6 | ||
|
|
505e6c67ce | ||
|
|
83e8a5e08c | ||
|
|
c82319a2c6 | ||
|
|
1fe7c697c5 | ||
|
|
239d1c60a3 | ||
|
|
bdfc96d56e | ||
|
|
8f7babd809 | ||
|
|
c708df574b | ||
|
|
401a511668 | ||
|
|
969879921e | ||
|
|
546ba9b45a | ||
|
|
27bdf54782 | ||
|
|
beed932e93 | ||
|
|
e5c4c08e66 | ||
|
|
2b201b0785 | ||
|
|
fdf5805911 | ||
|
|
9191075f48 | ||
|
|
9f73e4f865 | ||
|
|
1f20e028e1 | ||
|
|
da0fb17015 | ||
|
|
2aa71832b3 | ||
|
|
876def4392 | ||
|
|
0ba51b7270 | ||
|
|
7da4289097 | ||
|
|
ce5038cf08 | ||
|
|
1b0f53a88c | ||
|
|
d19992f0d3 | ||
|
|
a44d1ae668 | ||
|
|
14fd18479e | ||
|
|
1106a28288 | ||
|
|
f1a5d1b6a8 | ||
|
|
4193643306 | ||
|
|
ebe671ff5b | ||
|
|
fd447f2cc6 | ||
|
|
7dcd932ee5 | ||
|
|
ed233de565 | ||
|
|
a679573bf3 | ||
|
|
24d6892261 | ||
|
|
25dabd4955 | ||
|
|
ce65324c57 | ||
|
|
a0f10252a1 | ||
|
|
6e638488dc | ||
|
|
2e8de8c5c2 | ||
|
|
9c3aaf220a | ||
|
|
ab35bb4439 | ||
|
|
b48877b786 | ||
|
|
9fb85d4376 | ||
|
|
a582b15103 | ||
|
|
da3a51991c | ||
|
|
22f70c86f5 | ||
|
|
0daf4d895f | ||
|
|
80165a76e0 | ||
|
|
a6b025bdbf | ||
|
|
02a9ed1eb8 | ||
|
|
6f192b1712 | ||
|
|
b947a110e2 | ||
|
|
68885db538 | ||
|
|
0b54549c64 | ||
|
|
141bdccf02 | ||
|
|
4990effa1c | ||
|
|
2459f342c5 | ||
|
|
d68a2b7bce | ||
|
|
69d5ceac36 | ||
|
|
cf7039400c | ||
|
|
02a9d763cd |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -1,3 +1,6 @@
|
||||
dist/
|
||||
node_modules/
|
||||
typings/
|
||||
package-lock.json
|
||||
.idea/
|
||||
test/
|
||||
config.json
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
extras/
|
||||
node_modules/
|
||||
src/
|
||||
typings/
|
||||
ts.js
|
||||
tsconfig.json
|
||||
tsd.json
|
||||
tslint.json
|
||||
.idea/
|
||||
.github/
|
||||
test/
|
||||
config.json
|
||||
.travis.yml
|
||||
|
||||
@@ -3,6 +3,11 @@ sudo: false
|
||||
node_js:
|
||||
- 8
|
||||
- 9
|
||||
- 10
|
||||
- 11
|
||||
- 12
|
||||
# - 13
|
||||
# - 14
|
||||
before_install:
|
||||
- npm install --only=dev
|
||||
script:
|
||||
|
||||
122
README.md
122
README.md
@@ -1,6 +1,10 @@
|
||||
# Crunchy: a fork of Deathspike/CrunchyRoll.js
|
||||
|
||||
[](http://issuestats.com/github/Godzil/Crunchy) [](https://travis-ci.org/Godzil/Crunchy) [](https://codeclimate.com/github/Godzil/Crunchy/maintainability)
|
||||
[](https://travis-ci.org/Godzil/Crunchy)
|
||||
[](https://codeclimate.com/github/Godzil/Crunchy/maintainability)
|
||||

|
||||

|
||||

|
||||
|
||||
*Crunchy* is capable of downloading *anime* episodes from the popular *CrunchyRoll* streaming service. An episode is stored in the original video format (often H.264 in a MP4 container) and the configured subtitle format (ASS or SRT).The two output files are then merged into a single MKV file.
|
||||
|
||||
@@ -69,12 +73,13 @@ The [command-line interface](http://en.wikipedia.org/wiki/Command-line_interface
|
||||
-V, --version output the version number
|
||||
-p, --pass <s> The password.
|
||||
-u, --user <s> The e-mail address or username.
|
||||
-d, --unlog Unlog
|
||||
-c, --cache Disables the cache.
|
||||
-m, --merge Disables merging subtitles and videos.
|
||||
-e, --episodes <s> Episode list. Read documentation on how to use
|
||||
-f, --format <s> The subtitle format. (default: ass)
|
||||
-o, --output <s> The output path.
|
||||
-s, --series <s> The series name override.
|
||||
--ignoredub Experimental: Ignore all seasons where the title end with 'Dub)'
|
||||
-n, --nametmpl <s> Output name template (default: {SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - [{TAG}])
|
||||
-t, --tag <s> The subgroup. (default: CrunchyRoll)
|
||||
-r, --resolution <s> The video resolution. (valid: 360, 480, 720, 1080) (default: 1080)
|
||||
@@ -82,49 +87,125 @@ The [command-line interface](http://en.wikipedia.org/wiki/Command-line_interface
|
||||
--verbose Make tool verbose
|
||||
--rebuildcrp Rebuild the crpersistant file.
|
||||
--retry <i> Number or time to retry fetching an episode. (default: 5)
|
||||
-s, --sublang <items> Select the subtitle languages, multiple value separated by a comma are accepted (like: frFR,enUS )
|
||||
--sleepTime <i> Minimum wait time between each http requests.
|
||||
-h, --help output usage information
|
||||
|
||||
#### Batch-mode
|
||||
|
||||
When no sequence of series addresses is provided, the batch-mode source file will be read (which is *CrunchyRoll.txt* in the current work directory. Each line in this file is processed contain the URL of a series and can support some of the command line parameter (like `-e`). This makes it ideal to manage a large sequence of series addresses.
|
||||
|
||||
#### Configuration file
|
||||
|
||||
Starting from version 1.4.0, Crunchy store some information in a config.json file. The file which is use have to be in the folder you are calling Crunchy. This is partly by design and a limitation on where Crunchy can find files.
|
||||
|
||||
This file store some information like your username and password.
|
||||
|
||||
You don't need to create that file as Crunchy will create it for you, the first time you run it. Each run will update the content of the file, so it you run crunchy with your credential on the command line, it will add them to config file.
|
||||
|
||||
There are some parameter that the config file can accept which are not created by default, and some of them are cannont be set form the command line parameter.
|
||||
|
||||
Don't mess with them if you don't know what you are doing.
|
||||
|
||||
Here are the list of valid parameter in the config file:
|
||||
|
||||
- Output options
|
||||
* `merge` see `--merge`
|
||||
* `format` see `--format`
|
||||
* `output` see `--output`
|
||||
* `nametmpl` see `--nametmpl`
|
||||
* `tag` see `--tag`
|
||||
* `resolution` see `--resolution`
|
||||
* `sublang` see `--sublang`
|
||||
|
||||
- Login related options:
|
||||
* `pass` see `--user`
|
||||
* `user` see `--pass`
|
||||
* `userAgent` set the user agent reported by Crunchy while crawling pages
|
||||
* `logUsingApi`
|
||||
* `logUsingCookie`
|
||||
* `crSessionUrl`
|
||||
* `crDeviceType`
|
||||
* `crAPIVersion`
|
||||
* `crLocale`
|
||||
* `crSessionKey`
|
||||
* `crLoginUrl`
|
||||
|
||||
- Other options:
|
||||
* `sleepTime`: minimum delay (in ms) between each page load
|
||||
|
||||
- Generated values: don't touch them:
|
||||
* `crDeviceId`
|
||||
* `crSessionId`
|
||||
|
||||
Some of these login related options are not going to be documented on what to put there for _legal_ reason.
|
||||
|
||||
Crunchy will also create a `.cookie.jar` file in the output folder (by default the current folder) it is the file used by Crunchy to store the web cookies.
|
||||
|
||||
#### Examples
|
||||
|
||||
Download in batch-mode:
|
||||
|
||||
crunchy
|
||||
You will need to create the batch file (default name is `CrunchyRoll.txt`):
|
||||
|
||||
Download *Fairy Tail* to the current work directory:
|
||||
http://www.cr.com/tail-fairy
|
||||
http://www.cr.com/gin-mama
|
||||
http://www.cr.com/two-parts
|
||||
// Just download episodes 3 to 42
|
||||
http://www.cr.com/defense-of-dwarfs -e 3-42
|
||||
|
||||
crunchy http://www.crunchyroll.com/fairy-tail
|
||||
Then launch crunchy:
|
||||
|
||||
Download *Fairy Tail* to `C:\Anime`:
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy
|
||||
|
||||
crunchy --output C:\Anime http://www.crunchyroll.com/fairy-tail
|
||||
Download *Tail Fairy* to the current work directory:
|
||||
|
||||
Download episode 42 of *Fairy Tail* to `C:\Anime`:
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy
|
||||
|
||||
crunchy --output C:\Anime @http://www.crunchyroll.com/fairy-tail/episode-46-the-silver-labyrinth-662721
|
||||
Download *Tail Fairy* to `C:\Anime`:
|
||||
|
||||
crunchy -u login -p password --output C:\Anime http://www.cr.com/tail-fairy
|
||||
|
||||
Download episode 42 of *Tail Fairy* to `C:\Anime`:
|
||||
|
||||
crunchy -u login -p password --output C:\Anime @http://www.cr.com/tail-fairy/episode-42-the-episode-which-dont-exist-665544
|
||||
|
||||
*Notice the '@' in front of the URL, it is there to tell Crunchy that the URL is an episode URL and not a series URL.*
|
||||
|
||||
or
|
||||
|
||||
crunchy --output C:\Anime http://www.crunchyroll.com/fairy-tail -e 42
|
||||
crunchy -u login -p password --output C:\Anime http://www.cr.com/tail-fairy -e 42
|
||||
|
||||
Download episode 10 to 42 (both included) of *Fairy Tail*:
|
||||
Download episode 10 to 42 (both included) of *Tail Fairy*:
|
||||
|
||||
crunchy http://www.crunchyroll.com/fairy-tail -e 10-42
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy -e 10-42
|
||||
|
||||
Download episode up to 42 (included) of *Fairy Tail*:
|
||||
Download episode up to 42 (included) of *Tail Fairy*:
|
||||
|
||||
crunchy http://www.crunchyroll.com/fairy-tail -e -42
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy -e -42
|
||||
|
||||
Download episodes starting from 42 to the last available of *Fairy Tail*:
|
||||
Download episodes starting from 42 to the last available of *Tail Fairy*:
|
||||
|
||||
crunchy http://www.crunchyroll.com/fairy-tail -e 42-
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy -e 42-
|
||||
|
||||
Download episode up to 42 (included) of *Tail Fairy* with italian subtitles:
|
||||
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy -e -42 -s itIT
|
||||
|
||||
Download episode up to 42 (included) of *Tail Fairy* with italian subtitles and fallback to english if no available:
|
||||
|
||||
crunchy -u login -p password http://www.cr.com/tail-fairy -e -42 -s itIT,enUS
|
||||
|
||||
#### Known valid subtitles language:
|
||||
- `enUS` : English
|
||||
- `frFR` : French
|
||||
- `ptBR` : Portuguese (Brazil)
|
||||
- `esES` : Spanish (Spain)
|
||||
- `deDE` : German
|
||||
- `esLA` : Spanish (Latin America)
|
||||
- `itIT` : Italian
|
||||
- `arME` : Armenian
|
||||
- `ptPT` : Portuguese (Portugal)
|
||||
|
||||
#### Command line parameters
|
||||
|
||||
@@ -132,8 +213,12 @@ Download episodes starting from 42 to the last available of *Fairy Tail*:
|
||||
|
||||
* `-p or --pass <s>` sets the password.
|
||||
* `-u or --user <s>` sets the e-mail address or username.
|
||||
* `-d or --unlog` unlog
|
||||
|
||||
_Please remember that login has to be done for each call of Crunchy, as none of the credentials are stored_
|
||||
_New in 1.4.0_: Crunchy remember between run about login information and other, so you need to passe the login and password only once
|
||||
I recommend to unlog if you see some problems during the run.
|
||||
|
||||
*When you unlog, the cookie file is deleted as for some parameter in the config file (like username and password).*
|
||||
|
||||
##### Disables
|
||||
|
||||
@@ -149,6 +234,7 @@ Download episodes starting from 42 to the last available of *Fairy Tail*:
|
||||
* `-t or --tag <s>` sets The subgroup. (Default: CrunchyRoll)
|
||||
* `-r or --resolution <s>` sets the resolutoin you want to download (360, 480, 720, 1080)
|
||||
* `--retry <i>` set the number of try Crunchy will use if downloading a serie or episode fail
|
||||
* `--ignoredub` It is an experimental features that will ignore all season where the name ends with 'dub)'. The idea is to try to ignore dubbed season.
|
||||
|
||||
##### Others
|
||||
|
||||
@@ -180,5 +266,5 @@ More information will be added at a later point. For now the recommendations are
|
||||
|
||||
* Atom with `atom-typescript` and `linter-tslint` (and dependencies).
|
||||
|
||||
Since this project uses TypeScript, compile with `node run compile` to build the tool and `npm run test` to run the linter.
|
||||
Since this project uses TypeScript, compile with `node run build` to build the tool and `npm run test` to run the linter.
|
||||
|
||||
|
||||
1186
package-lock.json
generated
1186
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
45
package.json
45
package.json
@@ -15,36 +15,41 @@
|
||||
"engines": {
|
||||
"node": ">=5.0"
|
||||
},
|
||||
"version": "1.3.6",
|
||||
"version": "1.6.1",
|
||||
"bin": {
|
||||
"crunchy": "./bin/crunchy",
|
||||
"crunchy.sh": "./bin/crunchy.sh"
|
||||
},
|
||||
"dependencies": {
|
||||
"@types/node": "^10.3.3",
|
||||
"big-integer": "^1.6.31",
|
||||
"bluebird": "^3.5.1",
|
||||
"big-integer": "^1.6.48",
|
||||
"bluebird": "^3.7.2",
|
||||
"brotli": "^1.3.2",
|
||||
"cheerio": "^0.22.0",
|
||||
"cloudscraper": "^1.5.0",
|
||||
"commander": "^2.15.1",
|
||||
"fs-extra": "^6.0.1",
|
||||
"mkdirp": "^0.5.0",
|
||||
"cloudscraper": "^4.6.0",
|
||||
"commander": "^5.0.0",
|
||||
"fs-extra": "^9.0.0",
|
||||
"mkdirp": "^1.0.4",
|
||||
"pjson": "^1.0.9",
|
||||
"request": "^2.87.0",
|
||||
"request-promise": "^4.2.2",
|
||||
"xml2js": "^0.4.5"
|
||||
"request": "^2.88.2",
|
||||
"request-promise": "^4.2.5",
|
||||
"tough-cookie-file-store": "^1.2.0",
|
||||
"uuid": "^7.0.3",
|
||||
"xml2js": "^0.4.23"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bluebird": "^3.5.20",
|
||||
"@types/cheerio": "^0.22.7",
|
||||
"@types/fs-extra": "^5.0.3",
|
||||
"@types/mkdirp": "^0.5.2",
|
||||
"@types/request": "^2.47.1",
|
||||
"@types/request-promise": "^4.1.41",
|
||||
"@types/xml2js": "^0.4.3",
|
||||
"@types/bluebird": "^3.5.30",
|
||||
"@types/cheerio": "^0.22.17",
|
||||
"@types/fs-extra": "^8.1.0",
|
||||
"@types/mkdirp": "^1.0.0",
|
||||
"@types/node": "^13.11.1",
|
||||
"@types/request": "^2.48.4",
|
||||
"@types/request-promise": "^4.1.46",
|
||||
"@types/uuid": "^7.0.2",
|
||||
"@types/xml2js": "^0.4.5",
|
||||
"npm-check": "^5.9.2",
|
||||
"tsconfig-lint": "^0.12.0",
|
||||
"tslint": "^5.10.0",
|
||||
"typescript": "^2.9.2"
|
||||
"tslint": "^6.1.1",
|
||||
"typescript": "^3.8.3"
|
||||
},
|
||||
"scripts": {
|
||||
"prepublishOnly": "npm run build",
|
||||
|
||||
185
src/batch.ts
185
src/batch.ts
@@ -3,6 +3,8 @@ import commander = require('commander');
|
||||
import fs = require('fs');
|
||||
import path = require('path');
|
||||
import log = require('./log');
|
||||
import my_request = require('./my_request');
|
||||
import cfg = require('./config');
|
||||
import series from './series';
|
||||
|
||||
/* correspondances between resolution and value CR excpect */
|
||||
@@ -19,8 +21,32 @@ const resol_table: { [id: string]: IResolData; } =
|
||||
*/
|
||||
export default function(args: string[], done: (err?: Error) => void)
|
||||
{
|
||||
const config = parse(args);
|
||||
const batchPath = path.join(config.output || process.cwd(), config.batch);
|
||||
const config = Object.assign(cfg.load(), parse(args));
|
||||
let batchPath;
|
||||
|
||||
if (path.isAbsolute(config.batch))
|
||||
{
|
||||
batchPath = path.normalize(config.batch);
|
||||
}
|
||||
else
|
||||
{
|
||||
batchPath = path.normalize(path.join(process.cwd(), config.batch));
|
||||
}
|
||||
|
||||
if (config.nametmpl === undefined)
|
||||
{
|
||||
config.nametmpl = '{SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - {EPISODE_TITLE} - [{TAG}]';
|
||||
}
|
||||
|
||||
if (config.tag === undefined)
|
||||
{
|
||||
config.tag = 'CrunchyRoll';
|
||||
}
|
||||
|
||||
if (config.sublang === undefined)
|
||||
{
|
||||
config.sublang = [ 'enUS' ];
|
||||
}
|
||||
|
||||
// set resolution
|
||||
if (config.resolution)
|
||||
@@ -44,6 +70,21 @@ export default function(args: string[], done: (err?: Error) => void)
|
||||
config.video_quality = resol_table['1080'].quality;
|
||||
}
|
||||
|
||||
// Update the config file with new parameters
|
||||
cfg.save(config);
|
||||
|
||||
if (config.unlog)
|
||||
{
|
||||
config.crDeviceId = undefined;
|
||||
config.user = undefined;
|
||||
config.pass = undefined;
|
||||
my_request.eatCookies(config);
|
||||
cfg.save(config);
|
||||
log.info('Unlogged!');
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (config.debug)
|
||||
{
|
||||
/* Ugly but meh */
|
||||
@@ -62,12 +103,20 @@ export default function(args: string[], done: (err?: Error) => void)
|
||||
return done(err);
|
||||
}
|
||||
|
||||
if (!tasksArr || !tasksArr[0] || (tasksArr[0].address === ''))
|
||||
{
|
||||
return done();
|
||||
}
|
||||
|
||||
let i = 0;
|
||||
|
||||
(function next()
|
||||
{
|
||||
if (i >= tasksArr.length)
|
||||
{
|
||||
// Save configuration before leaving (should store info like session & other)
|
||||
cfg.save(config);
|
||||
|
||||
return done();
|
||||
}
|
||||
|
||||
@@ -80,12 +129,27 @@ export default function(args: string[], done: (err?: Error) => void)
|
||||
{
|
||||
if (errin)
|
||||
{
|
||||
if (tasksArr[i].retry <= 0)
|
||||
if (errin.error)
|
||||
{
|
||||
log.error(errin.stack || errin);
|
||||
/* Error from the request, so ignore it */
|
||||
tasksArr[i].retry = 0;
|
||||
}
|
||||
|
||||
if (errin.authError)
|
||||
{
|
||||
/* Force a graceful exit */
|
||||
log.error(errin.message);
|
||||
i = tasksArr.length;
|
||||
}
|
||||
else if (tasksArr[i].retry <= 0)
|
||||
{
|
||||
if (config.verbose)
|
||||
{
|
||||
log.error(JSON.stringify(errin));
|
||||
}
|
||||
if (config.debug)
|
||||
{
|
||||
log.dumpToDebug('BatchGiveUp', errin.stack || errin);
|
||||
log.dumpToDebug('BatchGiveUp', JSON.stringify(errin));
|
||||
}
|
||||
log.error('Cannot get episodes from "' + tasksArr[i].address + '", please rerun later');
|
||||
/* Go to the next on the list */
|
||||
@@ -95,11 +159,11 @@ export default function(args: string[], done: (err?: Error) => void)
|
||||
{
|
||||
if (config.verbose)
|
||||
{
|
||||
log.error(errin);
|
||||
log.error(JSON.stringify(errin));
|
||||
}
|
||||
if (config.debug)
|
||||
{
|
||||
log.dumpToDebug('BatchRetry', errin.stack || errin);
|
||||
log.dumpToDebug('BatchRetry', JSON.stringify(errin));
|
||||
}
|
||||
log.warn('Retrying to fetch episodes list from' + tasksArr[i].retry + ' / ' + config.retry);
|
||||
tasksArr[i].retry -= 1;
|
||||
@@ -109,7 +173,7 @@ export default function(args: string[], done: (err?: Error) => void)
|
||||
{
|
||||
i += 1;
|
||||
}
|
||||
next();
|
||||
setTimeout(next, config.sleepTime);
|
||||
});
|
||||
})();
|
||||
});
|
||||
@@ -149,7 +213,7 @@ function split(value: string): string[]
|
||||
return pieces;
|
||||
}
|
||||
|
||||
function get_min_filter(filter: string): number
|
||||
function get_min_filter(filter: string): IEpisodeNumber
|
||||
{
|
||||
if (filter !== undefined)
|
||||
{
|
||||
@@ -163,13 +227,41 @@ function get_min_filter(filter: string): number
|
||||
|
||||
if (tok[0] !== '')
|
||||
{
|
||||
return parseInt(tok[0], 10);
|
||||
/* If first item is not empty, ie '10-20' */
|
||||
if (tok[0].includes('e'))
|
||||
{
|
||||
/* include a e so we probably have something like 5e10
|
||||
aka season 5 episode 10
|
||||
*/
|
||||
const tok2 = tok[0].split('else');
|
||||
if (tok2.length > 2)
|
||||
{
|
||||
log.error('Invalid episode filter \'' + filter + '\'');
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
if (tok[0] !== '')
|
||||
{
|
||||
/* So season is properly filled */
|
||||
return {season: parseInt(tok2[0], 10), episode: parseInt(tok2[1], 10)};
|
||||
}
|
||||
else
|
||||
{
|
||||
/* we have 'e10' */
|
||||
return {season: 0, episode: parseInt(tok2[1], 10)};
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return {season: 0, episode: parseInt(tok[0], 10)};
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
/* First item is empty, ie '-20' */
|
||||
return {season: 0, episode: 0};
|
||||
}
|
||||
|
||||
function get_max_filter(filter: string): number
|
||||
function get_max_filter(filter: string): IEpisodeNumber
|
||||
{
|
||||
if (filter !== undefined)
|
||||
{
|
||||
@@ -184,15 +276,44 @@ function get_max_filter(filter: string): number
|
||||
if ((tok.length > 1) && (tok[1] !== ''))
|
||||
{
|
||||
/* We have a max value */
|
||||
return parseInt(tok[1], 10);
|
||||
return {season: +Infinity, episode: parseInt(tok[1], 10)};
|
||||
}
|
||||
else if ((tok.length === 1) && (tok[0] !== ''))
|
||||
{
|
||||
/* A single episode has been requested */
|
||||
return parseInt(tok[0], 10);
|
||||
return {season: +Infinity, episode: parseInt(tok[0], 10) + 1};
|
||||
}
|
||||
}
|
||||
return +Infinity;
|
||||
return {season: +Infinity, episode: +Infinity};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check that URL start with http:// or https://
|
||||
* As for some reason request just return an error but a useless one when that happen so check it
|
||||
* soon enough.
|
||||
*/
|
||||
function checkURL(address: string): boolean
|
||||
{
|
||||
if (address.startsWith('http:\/\/'))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if (address.startsWith('https:\/\/'))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if (address.startsWith('@http:\/\/'))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if (address.startsWith('@https:\/\/'))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
log.error('URL ' + address + ' miss \'http:\/\/\' or \'https:\/\/\' => will be ignored');
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -204,8 +325,13 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
|
||||
{
|
||||
return done(null, config.args.map((addressIn) =>
|
||||
{
|
||||
return {address: addressIn, retry: config.retry,
|
||||
episode_min: get_min_filter(config.episodes), episode_max: get_max_filter(config.episodes)};
|
||||
if (checkURL(addressIn))
|
||||
{
|
||||
return {address: addressIn, retry: config.retry,
|
||||
episode_min: get_min_filter(config.episodes), episode_max: get_max_filter(config.episodes)};
|
||||
}
|
||||
|
||||
return {address: '', retry: 0, episode_min: {season: 0, episode: 0}, episode_max: {season: 0, episode: 0}};
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -241,8 +367,11 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
|
||||
return;
|
||||
}
|
||||
|
||||
map.push({address: addressIn, retry: lineConfig.retry,
|
||||
episode_min: get_min_filter(lineConfig.episodes), episode_max: get_max_filter(lineConfig.episodes)});
|
||||
if (checkURL(addressIn))
|
||||
{
|
||||
map.push({address: addressIn, retry: lineConfig.retry,
|
||||
episode_min: get_min_filter(lineConfig.episodes), episode_max: get_max_filter(lineConfig.episodes)});
|
||||
}
|
||||
});
|
||||
});
|
||||
done(null, map);
|
||||
@@ -250,6 +379,10 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
|
||||
});
|
||||
}
|
||||
|
||||
function commaSeparatedList(value: any, dummyPrevious: any) {
|
||||
return value.split(',');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the arguments and returns a configuration.
|
||||
*/
|
||||
@@ -259,22 +392,28 @@ function parse(args: string[]): IConfigLine
|
||||
// Authentication
|
||||
.option('-p, --pass <s>', 'The password.')
|
||||
.option('-u, --user <s>', 'The e-mail address or username.')
|
||||
.option('-d, --unlog', 'Unlog')
|
||||
// Disables
|
||||
.option('-c, --cache', 'Disables the cache.')
|
||||
.option('-m, --merge', 'Disables merging subtitles and videos.')
|
||||
// Episode filter
|
||||
.option('-e, --episodes <s>', 'Episode list. Read documentation on how to use')
|
||||
// Settings
|
||||
.option('-l, --crlang <s>', 'CR page language (valid: en, fr, es, it, pt, de, ru).')
|
||||
.option('-s, --sublang <items>', 'Select the subtitle languages, multiple value separated by a comma ' +
|
||||
'are accepted (like: frFR,enUS )', commaSeparatedList)
|
||||
.option('-f, --format <s>', 'The subtitle format.', 'ass')
|
||||
.option('-o, --output <s>', 'The output path.')
|
||||
.option('-s, --series <s>', 'The series name override.')
|
||||
.option('-n, --nametmpl <s>', 'Output name template', '{SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - [{TAG}]')
|
||||
.option('-t, --tag <s>', 'The subgroup.', 'CrunchyRoll')
|
||||
.option('-r, --resolution <s>', 'The video resolution. (valid: 360, 480, 720, 1080)', '1080')
|
||||
.option('--ignoredub', 'Experimental: Ignore all seasons where the title end with \'Dub)\'')
|
||||
.option('-n, --nametmpl <s>', 'Output name template')
|
||||
.option('-t, --tag <s>', 'The subgroup.')
|
||||
.option('-r, --resolution <s>', 'The video resolution. (valid: 360, 480, 720, 1080)')
|
||||
.option('-b, --batch <s>', 'Batch file', 'CrunchyRoll.txt')
|
||||
.option('--verbose', 'Make tool verbose')
|
||||
.option('--debug', 'Create a debug file. Use only if requested!')
|
||||
.option('--rebuildcrp', 'Rebuild the crpersistant file.')
|
||||
.option('--retry <i>', 'Number or time to retry fetching an episode.', 5)
|
||||
.option('--retry <i>', 'Number or time to retry fetching an episode.', '5')
|
||||
.option('--sleepTime <i>', 'Minimum wait time between each http requests.')
|
||||
.parse(args);
|
||||
}
|
||||
|
||||
26
src/cli.ts
26
src/cli.ts
@@ -8,17 +8,27 @@ const current_version = pjson.version;
|
||||
|
||||
/* Check if the current version is the latest */
|
||||
log.info('Crunchy version ' + current_version);
|
||||
request.get({ uri: 'https://raw.githubusercontent.com/Godzil/Crunchy/master/package.json' },
|
||||
request.get({ uri: 'https://box.godzil.net/getVersion.php?tool=crunchy&v=' + current_version },
|
||||
(error: Error, response: any, body: any) =>
|
||||
{
|
||||
const onlinepkg = JSON.parse(body);
|
||||
let tmp = current_version.split('.');
|
||||
const cur = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
|
||||
tmp = onlinepkg.version.split('.');
|
||||
const dist = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
|
||||
if (dist > cur)
|
||||
if (response && (response.statusCode === 200))
|
||||
{
|
||||
log.warn('There is a newer version of crunchy (v' + onlinepkg.version + '), you should update!');
|
||||
const onlinepkg = JSON.parse(body);
|
||||
if (onlinepkg.status === 'ok')
|
||||
{
|
||||
let tmp = current_version.split('.');
|
||||
const cur = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
|
||||
tmp = onlinepkg.version.split('.');
|
||||
const dist = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
|
||||
if (dist > cur)
|
||||
{
|
||||
log.warnMore('There is a newer version of crunchy (v' + onlinepkg.version + '), you should update!');
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
log.info('Error while checking for the current version.');
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
82
src/config.ts
Normal file
82
src/config.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
'use strict';
|
||||
import os = require('os');
|
||||
import fs = require('fs-extra');
|
||||
import path = require('path');
|
||||
|
||||
const configFile = path.join(process.cwd(), 'config.json');
|
||||
|
||||
function fileExist(path: string)
|
||||
{
|
||||
try
|
||||
{
|
||||
fs.statSync(path);
|
||||
return true;
|
||||
} catch (e)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function load(): IConfigLine
|
||||
{
|
||||
if (fileExist(configFile))
|
||||
{
|
||||
const data = fs.readFileSync(configFile, 'utf8');
|
||||
return JSON.parse(data);
|
||||
}
|
||||
|
||||
return {args: undefined};
|
||||
}
|
||||
|
||||
export function save(config: IConfig)
|
||||
{
|
||||
const tmp = JSON.parse(JSON.stringify(config));
|
||||
|
||||
// Things added by the command line parser
|
||||
tmp.rawArgs = undefined;
|
||||
tmp.options = undefined;
|
||||
tmp._execs = undefined;
|
||||
tmp._args = undefined;
|
||||
tmp._name = undefined;
|
||||
tmp._version = undefined;
|
||||
tmp._versionOptionName = undefined;
|
||||
tmp._events = undefined;
|
||||
tmp._eventsCount = undefined;
|
||||
tmp.args = undefined;
|
||||
tmp.commands = undefined;
|
||||
tmp._allowUnknownOption = undefined;
|
||||
tmp.parent = undefined;
|
||||
tmp._scriptPath = undefined;
|
||||
tmp._optionValues = undefined;
|
||||
tmp._storeOptionsAsProperties = undefined;
|
||||
tmp._passCommandToAction = undefined;
|
||||
tmp._actionResults = undefined;
|
||||
tmp._actionHandler = undefined;
|
||||
tmp._executableHandler = undefined;
|
||||
tmp._executableFile = undefined;
|
||||
tmp._defaultCommandName = undefined;
|
||||
tmp._exitCallback = undefined;
|
||||
tmp._alias = undefined;
|
||||
tmp._noHelp = undefined;
|
||||
tmp._helpFlags = undefined;
|
||||
tmp._helpDescription = undefined;
|
||||
tmp._helpShortFlag = undefined;
|
||||
tmp._helpLongFlag = undefined;
|
||||
tmp._hasImplicitHelpCommand = undefined;
|
||||
tmp._helpCommandName = undefined;
|
||||
tmp._helpCommandnameAndArgs = undefined;
|
||||
tmp._helpCommandDescription = undefined;
|
||||
|
||||
// Things we don't want to save
|
||||
tmp.cache = undefined;
|
||||
tmp.episodes = undefined;
|
||||
tmp.series = undefined;
|
||||
tmp.rebuildcrp = undefined;
|
||||
tmp.batch = undefined;
|
||||
tmp.verbose = undefined;
|
||||
tmp.debug = undefined;
|
||||
tmp.unlog = undefined;
|
||||
tmp.ignoredub = undefined;
|
||||
|
||||
fs.writeFileSync(configFile, JSON.stringify(tmp, null, ' '));
|
||||
}
|
||||
307
src/episode.ts
307
src/episode.ts
@@ -5,6 +5,7 @@ import mkdirp = require('mkdirp');
|
||||
import my_request = require('./my_request');
|
||||
import path = require('path');
|
||||
import subtitle from './subtitle/index';
|
||||
import vlos from './vlos';
|
||||
import video from './video/index';
|
||||
import xml2js = require('xml2js');
|
||||
import log = require('./log');
|
||||
@@ -21,15 +22,24 @@ export default function(config: IConfig, address: string, done: (err: Error, ign
|
||||
return done(err, false);
|
||||
}
|
||||
|
||||
scrapePlayer(config, address, page.id, (errS, player) =>
|
||||
if (page.media != null)
|
||||
{
|
||||
if (errS)
|
||||
/* No player to scrape */
|
||||
download(config, page, null, done);
|
||||
}
|
||||
else
|
||||
{
|
||||
/* The old way */
|
||||
scrapePlayer(config, address, page.id, (errS, player) =>
|
||||
{
|
||||
return done(errS, false);
|
||||
}
|
||||
if (errS)
|
||||
{
|
||||
return done(errS, false);
|
||||
}
|
||||
|
||||
download(config, page, player, done);
|
||||
});
|
||||
download(config, page, player, done);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -66,13 +76,15 @@ function fileExist(path: string)
|
||||
|
||||
function sanitiseFileName(str: string)
|
||||
{
|
||||
return str.replace(/[\/':\?\*"<>\.]/g, '_');
|
||||
const sanitized = str.replace(/[\/':\?\*"<>\\\.\|]/g, '_');
|
||||
|
||||
return sanitized.replace(/{DIR_SEPARATOR}/g, '/');
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads the subtitle and video.
|
||||
*/
|
||||
function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void)
|
||||
function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error | string, ign: boolean) => void)
|
||||
{
|
||||
const serieFolder = sanitiseFileName(config.series || page.series);
|
||||
|
||||
@@ -109,16 +121,11 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
|
||||
return done(null, true);
|
||||
}
|
||||
|
||||
mkdirp(path.dirname(filePath), (errM: Error) =>
|
||||
const ret = mkdirp(path.dirname(filePath));
|
||||
if (ret)
|
||||
{
|
||||
if (errM)
|
||||
{
|
||||
log.dispEpisode(fileName, 'Error...', true);
|
||||
return done(errM, false);
|
||||
}
|
||||
|
||||
log.dispEpisode(fileName, 'Fetching...', false);
|
||||
downloadSubtitle(config, player, filePath, (errDS) =>
|
||||
downloadSubtitle(config, page, player, filePath, (errDS) =>
|
||||
{
|
||||
if (errDS)
|
||||
{
|
||||
@@ -127,7 +134,8 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (player.video.file !== undefined)
|
||||
if ( ((page.media === null) && (player.video.file !== undefined))
|
||||
|| ((page.media !== null) /* Do they still create page in advance for unreleased episodes? */) )
|
||||
{
|
||||
log.dispEpisode(fileName, 'Fetching video...', false);
|
||||
downloadVideo(config, page, player, filePath, (errDV) =>
|
||||
@@ -143,10 +151,28 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
|
||||
return complete(fileName, 'Finished!', now, done);
|
||||
}
|
||||
|
||||
const isSubtited = Boolean(player.subtitle);
|
||||
let isSubtitled = true;
|
||||
|
||||
if (page.media === null)
|
||||
{
|
||||
isSubtitled = Boolean(player.subtitle);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (page.media.subtitles.length === 0)
|
||||
{
|
||||
isSubtitled = false;
|
||||
}
|
||||
}
|
||||
|
||||
let videoExt = '.mp4';
|
||||
if ( (page.media === null) && (player.video.mode === 'RTMP'))
|
||||
{
|
||||
videoExt = path.extname(player.video.file);
|
||||
}
|
||||
|
||||
log.dispEpisode(fileName, 'Merging...', false);
|
||||
video.merge(config, isSubtited, player.video.file, filePath, player.video.mode, config.verbose, (errVM) =>
|
||||
video.merge(config, isSubtitled, videoExt, filePath, config.verbose, (errVM) =>
|
||||
{
|
||||
if (errVM)
|
||||
{
|
||||
@@ -164,51 +190,136 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
|
||||
done(null, true);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
log.dispEpisode(fileName, 'Error creating folder \'' + filePath + '\'...', true);
|
||||
return done('Cannot create folder', false);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the subtitles to disk.
|
||||
*/
|
||||
function downloadSubtitle(config: IConfig, player: IEpisodePlayer, filePath: string, done: (err?: Error) => void)
|
||||
function downloadSubtitle(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
|
||||
filePath: string, done: (err?: Error | string) => void)
|
||||
{
|
||||
const enc = player.subtitle;
|
||||
|
||||
if (!enc)
|
||||
if (page.media !== null)
|
||||
{
|
||||
return done();
|
||||
}
|
||||
|
||||
subtitle.decode(enc.id, enc.iv, enc.data, (errSD, data) =>
|
||||
{
|
||||
if (errSD)
|
||||
const subs = page.media.subtitles;
|
||||
if (subs.length === 0)
|
||||
{
|
||||
return done(errSD);
|
||||
/* No downloadable subtitles */
|
||||
console.warn('Can\'t find subtitle ?!');
|
||||
return done();
|
||||
}
|
||||
|
||||
const formats = subtitle.formats;
|
||||
const format = formats[config.format] ? config.format : 'ass';
|
||||
let i;
|
||||
let j;
|
||||
|
||||
formats[format](config, data, (errF: Error, decodedSubtitle: string) =>
|
||||
/* Find a proper subtitles */
|
||||
for (j = 0; j < config.sublang.length; j++)
|
||||
{
|
||||
if (errF)
|
||||
const reqSubLang = config.sublang[j];
|
||||
for (i = 0; i < subs.length; i++)
|
||||
{
|
||||
return done(errF);
|
||||
const curSub = subs[i];
|
||||
if (curSub.format === 'ass' && curSub.language === reqSubLang)
|
||||
{
|
||||
my_request.get(config, curSub.url, (err, result) =>
|
||||
{
|
||||
if (err)
|
||||
{
|
||||
log.error('An error occured while fetching subtitles...');
|
||||
return done(err);
|
||||
}
|
||||
|
||||
fs.writeFile(filePath + '.ass', '\ufeff' + result, done);
|
||||
});
|
||||
|
||||
/* Break from the first loop */
|
||||
j = config.sublang.length;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (i >= subs.length)
|
||||
{
|
||||
done('Cannot find subtitles with requested language(s)');
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
const enc = player.subtitle;
|
||||
|
||||
if (!enc)
|
||||
{
|
||||
return done();
|
||||
}
|
||||
|
||||
subtitle.decode(enc.id, enc.iv, enc.data, (errSD, data) =>
|
||||
{
|
||||
if (errSD)
|
||||
{
|
||||
log.error('An error occured while getting subtitles...');
|
||||
return done(errSD);
|
||||
}
|
||||
|
||||
fs.writeFile(filePath + '.' + format, '\ufeff' + decodedSubtitle, done);
|
||||
if (config.debug)
|
||||
{
|
||||
log.dumpToDebug('SubtitlesXML', data);
|
||||
}
|
||||
|
||||
const formats = subtitle.formats;
|
||||
const format = formats[config.format] ? config.format : 'ass';
|
||||
|
||||
formats[format](config, data, (errF: Error, decodedSubtitle: string) =>
|
||||
{
|
||||
if (errF)
|
||||
{
|
||||
return done(errF);
|
||||
}
|
||||
|
||||
fs.writeFile(filePath + '.' + format, '\ufeff' + decodedSubtitle, done);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Streams the video to disk.
|
||||
*/
|
||||
function downloadVideo(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
|
||||
filePath: string, done: (err: Error) => void)
|
||||
filePath: string, done: (err: any) => void)
|
||||
{
|
||||
video.stream(player.video.host, player.video.file, page.swf, filePath,
|
||||
path.extname(player.video.file), player.video.mode, config.verbose, done);
|
||||
if (player == null)
|
||||
{
|
||||
/* new way */
|
||||
|
||||
const streams = page.media.streams;
|
||||
let i;
|
||||
/* Find a proper subtitles */
|
||||
for (i = 0; i < streams.length; i++)
|
||||
{
|
||||
if (streams[i].format === 'vo_adaptive_hls' && streams[i].audio_lang === 'jaJP' &&
|
||||
streams[i].hardsub_lang === null)
|
||||
{
|
||||
video.stream('', streams[i].url, '', filePath,
|
||||
'mp4', 'HLS', config.verbose, done);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (i >= streams.length)
|
||||
{
|
||||
done('Cannot find a valid stream');
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Old way */
|
||||
video.stream(player.video.host, player.video.file, page.swf, filePath,
|
||||
path.extname(player.video.file), player.video.mode, config.verbose, done);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -270,51 +381,73 @@ function scrapePage(config: IConfig, address: string, done: (err: Error, page?:
|
||||
}
|
||||
|
||||
const $ = cheerio.load(result);
|
||||
const swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href'));
|
||||
const regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][\-0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/;
|
||||
const look = $('#showmedia_about_media').text();
|
||||
const seasonTitle = $('span[itemprop="title"]').text();
|
||||
const episodeTitle = $('#showmedia_about_name').text().replace(/[“”]/g, '');
|
||||
const data = regexp.exec(look);
|
||||
/* First check if we have the new player */
|
||||
const vlosScript = $('#vilos-iframe-container');
|
||||
|
||||
if (config.debug)
|
||||
if (vlosScript)
|
||||
{
|
||||
log.dumpToDebug('episode page', $.html());
|
||||
}
|
||||
const pageMetadata = JSON.parse($('script[type="application/ld+json"]')[0].children[0].data);
|
||||
const divScript = $('div[id="showmedia_video_box_wide"]');
|
||||
const scripts = divScript.find('script').toArray();
|
||||
const script = scripts[2].children[0].data;
|
||||
let seasonNumber = '1';
|
||||
let seasonTitle = '';
|
||||
|
||||
if (!swf || !data)
|
||||
{
|
||||
log.warn('Somethig unexpected in the page at ' + address + ' (data are: ' + look + ')');
|
||||
log.warn('Setting Season to ’0’ and episode to ’0’...');
|
||||
|
||||
if (config.debug)
|
||||
if (pageMetadata.partOfSeason)
|
||||
{
|
||||
log.dumpToDebug('episode unexpected', look);
|
||||
}
|
||||
seasonNumber = pageMetadata.partOfSeason.seasonNumber;
|
||||
if (seasonNumber === '0') { seasonNumber = '1'; }
|
||||
|
||||
done(null, {
|
||||
episode: '0',
|
||||
id: epId,
|
||||
series: seasonTitle,
|
||||
season: seasonTitle,
|
||||
title: episodeTitle,
|
||||
swf: swf[1],
|
||||
volume: '0',
|
||||
filename: '',
|
||||
});
|
||||
seasonTitle = pageMetadata.partOfSeason.name;
|
||||
}
|
||||
done(null, vlos.getMedia(script, seasonTitle, seasonNumber));
|
||||
}
|
||||
else
|
||||
{
|
||||
done(null, {
|
||||
episode: data[3],
|
||||
id: epId,
|
||||
series: data[1],
|
||||
season: seasonTitle,
|
||||
title: episodeTitle,
|
||||
swf: swf[1],
|
||||
volume: data[2] || '1',
|
||||
filename: '',
|
||||
});
|
||||
/* Use the old way */
|
||||
const swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href'));
|
||||
const regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][\-0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/;
|
||||
const seasonTitle = $('span[itemprop="title"]').text();
|
||||
const look = $('#showmedia_about_media').text();
|
||||
const episodeTitle = $('#showmedia_about_name').text().replace(/[“”]/g, '');
|
||||
const data = regexp.exec(look);
|
||||
|
||||
if (config.debug) {
|
||||
log.dumpToDebug('episode page', $.html());
|
||||
}
|
||||
|
||||
if (!swf || !data) {
|
||||
log.warn('Somethig unexpected in the page at ' + address + ' (data are: ' + look + ')');
|
||||
log.warn('Setting Season to ’0’ and episode to ’0’...');
|
||||
|
||||
if (config.debug) {
|
||||
log.dumpToDebug('episode unexpected', look);
|
||||
}
|
||||
|
||||
done(null, {
|
||||
episode: '0',
|
||||
id: epId,
|
||||
series: seasonTitle,
|
||||
season: seasonTitle,
|
||||
title: episodeTitle,
|
||||
swf: swf[1],
|
||||
volume: '0',
|
||||
filename: '',
|
||||
media: null,
|
||||
});
|
||||
} else {
|
||||
done(null, {
|
||||
episode: data[3],
|
||||
id: epId,
|
||||
series: data[1],
|
||||
season: seasonTitle,
|
||||
title: episodeTitle,
|
||||
swf: swf[1],
|
||||
volume: data[2] || '1',
|
||||
filename: '',
|
||||
media: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -331,15 +464,15 @@ function scrapePlayer(config: IConfig, address: string, id: number, done: (err:
|
||||
return done(new Error('Invalid address.'));
|
||||
}
|
||||
|
||||
my_request.post(config, {
|
||||
form: {
|
||||
current_page: address,
|
||||
video_format: config.video_format,
|
||||
video_quality: config.video_quality,
|
||||
media_id: id
|
||||
},
|
||||
url: url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id,
|
||||
}, (err, result) =>
|
||||
const postForm = {
|
||||
current_page: address,
|
||||
video_format: config.video_format,
|
||||
video_quality: config.video_quality,
|
||||
media_id: id
|
||||
};
|
||||
|
||||
my_request.post(config, url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id, postForm,
|
||||
(err, result) =>
|
||||
{
|
||||
if (err)
|
||||
{
|
||||
|
||||
5
src/interface/AuthError.d.ts
vendored
Normal file
5
src/interface/AuthError.d.ts
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
interface IAuthError extends Error {
|
||||
name: string;
|
||||
message: string;
|
||||
authError: boolean;
|
||||
}
|
||||
18
src/interface/IConfig.d.ts
vendored
18
src/interface/IConfig.d.ts
vendored
@@ -7,11 +7,14 @@ interface IConfig {
|
||||
merge?: boolean;
|
||||
episodes?: string;
|
||||
// Settings
|
||||
crlang?: string;
|
||||
sublang?: any;
|
||||
format?: string;
|
||||
output?: string;
|
||||
series?: string;
|
||||
nametmpl?: string;
|
||||
tag?: string;
|
||||
ignoredub?: boolean;
|
||||
resolution?: string;
|
||||
video_format?: string;
|
||||
video_quality?: string;
|
||||
@@ -19,5 +22,20 @@ interface IConfig {
|
||||
batch?: string;
|
||||
verbose?: boolean;
|
||||
debug?: boolean;
|
||||
unlog?: boolean;
|
||||
retry?: number;
|
||||
sleepTime?: number;
|
||||
// Login options
|
||||
userAgent?: string;
|
||||
logUsingApi?: boolean;
|
||||
logUsingCookie?: boolean;
|
||||
crSessionUrl?: string;
|
||||
crDeviceType?: string;
|
||||
crAPIVersion?: string;
|
||||
crLocale?: string;
|
||||
crSessionKey?: string;
|
||||
crLoginUrl?: string;
|
||||
// Generated values
|
||||
crDeviceId?: string;
|
||||
crSessionId?: string;
|
||||
}
|
||||
|
||||
4
src/interface/IConfigTask.d.ts
vendored
4
src/interface/IConfigTask.d.ts
vendored
@@ -1,6 +1,6 @@
|
||||
interface IConfigTask {
|
||||
address: string;
|
||||
retry: number;
|
||||
episode_min: number;
|
||||
episode_max: number;
|
||||
episode_min: IEpisodeNumber;
|
||||
episode_max: IEpisodeNumber;
|
||||
}
|
||||
|
||||
4
src/interface/IEpisodeNumber.d.ts
vendored
Normal file
4
src/interface/IEpisodeNumber.d.ts
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
interface IEpisodeNumber {
|
||||
season: number,
|
||||
episode: number
|
||||
}
|
||||
1
src/interface/IEpisodePage.d.ts
vendored
1
src/interface/IEpisodePage.d.ts
vendored
@@ -7,4 +7,5 @@ interface IEpisodePage {
|
||||
title: string;
|
||||
swf: string;
|
||||
filename: string;
|
||||
media: IVlosScript;
|
||||
}
|
||||
|
||||
1
src/interface/ISeriesEpisode.d.ts
vendored
1
src/interface/ISeriesEpisode.d.ts
vendored
@@ -1,6 +1,7 @@
|
||||
interface ISeriesEpisode {
|
||||
address: string;
|
||||
episode: string;
|
||||
seasonName: string;
|
||||
volume: number;
|
||||
retry: number;
|
||||
}
|
||||
|
||||
14
src/interface/IVlosScript.d.ts
vendored
Normal file
14
src/interface/IVlosScript.d.ts
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
interface IVlosScript
|
||||
{
|
||||
metadata: {
|
||||
episode_number: any;
|
||||
id: any;
|
||||
title: any;
|
||||
};
|
||||
confic: any;
|
||||
subtitles: any;
|
||||
streams: any;
|
||||
series: {
|
||||
title: any;
|
||||
};
|
||||
}
|
||||
66
src/languages.ts
Normal file
66
src/languages.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
'use strict';
|
||||
|
||||
const localeCC: { [id: string]: string; } =
|
||||
{
|
||||
enUS: 'en', enGB: 'en',
|
||||
esLA: 'es', esES: 'es',
|
||||
ptPT: 'pt', ptBR: 'pt',
|
||||
frFR: 'fr',
|
||||
deDE: 'de',
|
||||
itIT: 'it',
|
||||
ruRU: 'ru',
|
||||
};
|
||||
|
||||
export function localeToCC(locale: string): string
|
||||
{
|
||||
let ret = localeCC.enGB;
|
||||
|
||||
if (locale in localeCC)
|
||||
{
|
||||
ret = localeCC[locale];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
const dubignore_regexp: { [id: string]: RegExp; } =
|
||||
{
|
||||
en: /\(.*Dub(?:bed)?.*\)|(?:\(RU\))|\(Russian\)/i,
|
||||
fr: /\(.*Dub(?:bed)?.*\)|(?:\(RU\))|\(?Doublage.*\)|\(Russian\)?/,
|
||||
de: /\(.*isch\)|\(Dubbed\)|\(RU\)|\(Russian\)/
|
||||
};
|
||||
|
||||
export function get_diregexp(config: IConfig): RegExp
|
||||
{
|
||||
let ret = dubignore_regexp.en;
|
||||
|
||||
if (config.crlang in dubignore_regexp)
|
||||
{
|
||||
ret = dubignore_regexp[config.crlang];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
const episodes_regexp: { [id: string]: RegExp; } =
|
||||
{
|
||||
en: /Episode\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
fr: /Épisode\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
de: /Folge\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
es: /Episodio\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
it: /Episodio\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
pt: /Episódio\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
ru: /Серия\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
|
||||
};
|
||||
|
||||
export function get_epregexp(config: IConfig): RegExp
|
||||
{
|
||||
let ret = episodes_regexp.en;
|
||||
|
||||
if (config.crlang in episodes_regexp)
|
||||
{
|
||||
ret = episodes_regexp[config.crlang];
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
18
src/log.ts
18
src/log.ts
@@ -26,6 +26,12 @@ export function warn(str: string)
|
||||
console.log(' \x1B[1;33m* WARN \x1B[0m: ' + str);
|
||||
}
|
||||
|
||||
export function warnMore(str: string)
|
||||
{
|
||||
/* Do fancy output */
|
||||
console.log(' \x1B[1;38;5;166m* WARN \x1B[0m: ' + str);
|
||||
}
|
||||
|
||||
export function dispEpisode(name: string, status: string, addNL: boolean)
|
||||
{
|
||||
/* Do fancy output */
|
||||
@@ -41,15 +47,9 @@ export function dumpToDebug(what: string, data: any, create = false)
|
||||
{
|
||||
if (create)
|
||||
{
|
||||
fs.writeFile('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n', (err) =>
|
||||
{
|
||||
if (err) throw err;
|
||||
});
|
||||
fs.writeFileSync('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n');
|
||||
return;
|
||||
}
|
||||
|
||||
fs.appendFile('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n', (err) =>
|
||||
{
|
||||
if (err) throw err;
|
||||
});
|
||||
}
|
||||
fs.appendFileSync('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n');
|
||||
}
|
||||
|
||||
@@ -3,72 +3,77 @@ import cheerio = require('cheerio');
|
||||
import request = require('request');
|
||||
import rp = require('request-promise');
|
||||
import Promise = require('bluebird');
|
||||
import uuid = require('uuid');
|
||||
import path = require('path');
|
||||
import fs = require('fs-extra');
|
||||
import languages = require('./languages');
|
||||
import log = require('./log');
|
||||
|
||||
import { RequestPromise } from 'request-promise';
|
||||
import { Response } from 'request';
|
||||
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const cloudscraper = require('cloudscraper');
|
||||
const cookieStore = require('tough-cookie-file-store');
|
||||
|
||||
const CR_COOKIE_DOMAIN = 'http://crunchyroll.com';
|
||||
|
||||
let isAuthenticated = false;
|
||||
let isPremium = false;
|
||||
|
||||
const defaultHeaders: request.Headers =
|
||||
let j: request.CookieJar;
|
||||
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
import cloudscraper = require('cloudscraper');
|
||||
let currentOptions: any;
|
||||
let optionsSet = false;
|
||||
|
||||
function AuthError(msg: string): IAuthError
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64; x64; rv:58.0) Gecko/20100101 Firefox/58.0',
|
||||
'Connection': 'keep-alive',
|
||||
'Referer': 'https://www.crunchyroll.com/login',
|
||||
};
|
||||
|
||||
function generateDeviceId(): string
|
||||
{
|
||||
let id = '';
|
||||
const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
|
||||
|
||||
for (let i = 0; i < 32; i++)
|
||||
{
|
||||
id += possible.charAt(Math.floor(Math.random() * possible.length));
|
||||
}
|
||||
|
||||
return id;
|
||||
return { name: 'AuthError', message: msg, authError: true };
|
||||
}
|
||||
|
||||
function startSession(): Promise<string>
|
||||
function startSession(config: IConfig): Promise<any>
|
||||
{
|
||||
return rp(
|
||||
{
|
||||
method: 'GET',
|
||||
url: 'CR_SESSION_URL',
|
||||
url: config.crSessionUrl,
|
||||
qs:
|
||||
{
|
||||
device_id: generateDeviceId(),
|
||||
device_type: 'CR_DEVICE_TYPE',
|
||||
access_token: 'CR_SESSION_KEY',
|
||||
version: 'CR_API_VERSION',
|
||||
locale: 'CR_LOCALE',
|
||||
device_id: config.crDeviceId,
|
||||
device_type: config.crDeviceType,
|
||||
access_token: config.crSessionKey,
|
||||
version: config.crAPIVersion,
|
||||
locale: config.crLocale,
|
||||
},
|
||||
json: true,
|
||||
})
|
||||
.then((response: any) =>
|
||||
{
|
||||
if ((response.data === undefined) || (response.data.session_id === undefined))
|
||||
{
|
||||
throw new Error('Getting session failed: ' + JSON.stringify(response));
|
||||
}
|
||||
|
||||
return response.data.session_id;
|
||||
});
|
||||
}
|
||||
|
||||
function login(sessionId: string, user: string, pass: string): Promise<any>
|
||||
function APIlogin(config: IConfig, sessionId: string, user: string, pass: string): Promise<any>
|
||||
{
|
||||
return rp(
|
||||
{
|
||||
method: 'POST',
|
||||
url: 'CR_LOGIN_URL',
|
||||
url: config.crLoginUrl,
|
||||
form:
|
||||
{
|
||||
account: user,
|
||||
password: pass,
|
||||
session_id: sessionId,
|
||||
version: 'CR_API_VERSION',
|
||||
version: config.crAPIVersion,
|
||||
},
|
||||
json: true,
|
||||
jar: j,
|
||||
})
|
||||
.then((response) =>
|
||||
{
|
||||
@@ -77,12 +82,114 @@ function login(sessionId: string, user: string, pass: string): Promise<any>
|
||||
});
|
||||
}
|
||||
|
||||
// TODO: logout
|
||||
function checkIfUserIsAuth(config: IConfig, done: (err: any) => void): void
|
||||
{
|
||||
/**
|
||||
* The main page give us some information about the user
|
||||
*/
|
||||
const url = 'http://www.crunchyroll.com/';
|
||||
|
||||
cloudscraper.get(url, getOptions(config, null), (err: any, rep: Response, body: string) =>
|
||||
{
|
||||
if (err)
|
||||
{
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
/* As we are here, try to detect which locale CR tell us */
|
||||
const localeRE = /LOCALE = "([a-zA-Z]+)",/g;
|
||||
const locale = localeRE.exec($('script').text())[1];
|
||||
const countryCode = languages.localeToCC(locale);
|
||||
|
||||
if (config.crlang === undefined)
|
||||
{
|
||||
log.info('No locale set. Setting to the one reported by CR: "' + countryCode + '"');
|
||||
config.crlang = countryCode;
|
||||
}
|
||||
else if (config.crlang !== countryCode)
|
||||
{
|
||||
log.warn('Crunchy is configured for locale "' + config.crlang + '" but CR report "' + countryCode + '" (LOCALE = ' + locale + ')');
|
||||
log.warn('Check if it is correct or rerun (once) with "-l ' + countryCode + '" to correct.');
|
||||
}
|
||||
|
||||
/* Check if auth worked */
|
||||
const regexps = /ga\('set', 'dimension[5-8]', '([^']*)'\);/g;
|
||||
const dims = regexps.exec($('script').text());
|
||||
|
||||
for (let i = 1; i < 5; i++)
|
||||
{
|
||||
if ((dims[i] !== undefined) && (dims[i] !== '') && (dims[i] !== 'not-registered'))
|
||||
{
|
||||
isAuthenticated = true;
|
||||
}
|
||||
|
||||
if ((dims[i] === 'premium') || (dims[i] === 'premiumplus'))
|
||||
{
|
||||
isPremium = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (isAuthenticated === false)
|
||||
{
|
||||
const error = $('ul.message, li.error').text();
|
||||
log.warn('Authentication failed: ' + error);
|
||||
|
||||
log.dumpToDebug('not auth rep', rep);
|
||||
log.dumpToDebug('not auth body', body);
|
||||
|
||||
return done(AuthError('Authentication failed: ' + error));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (isPremium === false)
|
||||
{
|
||||
log.warn('Do not use this app without a premium account.');
|
||||
}
|
||||
else
|
||||
{
|
||||
log.info('You have a premium account! Good!');
|
||||
}
|
||||
}
|
||||
|
||||
done(null);
|
||||
});
|
||||
}
|
||||
|
||||
function loadCookies(config: IConfig)
|
||||
{
|
||||
const cookiePath = path.join(config.output || process.cwd(), '.cookies.json');
|
||||
|
||||
if (!fs.existsSync(cookiePath))
|
||||
{
|
||||
fs.closeSync(fs.openSync(cookiePath, 'w'));
|
||||
}
|
||||
|
||||
j = request.jar(new cookieStore(cookiePath));
|
||||
}
|
||||
|
||||
export function eatCookies(config: IConfig)
|
||||
{
|
||||
const cookiePath = path.join(config.output || process.cwd(), '.cookies.json');
|
||||
|
||||
if (fs.existsSync(cookiePath))
|
||||
{
|
||||
fs.removeSync(cookiePath);
|
||||
}
|
||||
|
||||
j = undefined;
|
||||
}
|
||||
|
||||
export function getUserAgent(): string
|
||||
{
|
||||
return currentOptions.headers['User-Agent'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs a GET request for the resource.
|
||||
*/
|
||||
export function get(config: IConfig, options: string|request.Options, done: (err: Error, result?: string) => void)
|
||||
export function get(config: IConfig, url: string, done: (err: any, result?: string) => void)
|
||||
{
|
||||
authenticate(config, (err) =>
|
||||
{
|
||||
@@ -91,9 +198,10 @@ export function get(config: IConfig, options: string|request.Options, done: (err
|
||||
return done(err);
|
||||
}
|
||||
|
||||
cloudscraper.request(modify(options, 'GET'), (error: Error, response: any, body: any) =>
|
||||
cloudscraper.get(url, getOptions(config, null), (error: any, response: any, body: any) =>
|
||||
{
|
||||
if (error) return done(error);
|
||||
|
||||
done(null, typeof body === 'string' ? body : String(body));
|
||||
});
|
||||
});
|
||||
@@ -102,125 +210,215 @@ export function get(config: IConfig, options: string|request.Options, done: (err
|
||||
/**
|
||||
* Performs a POST request for the resource.
|
||||
*/
|
||||
export function post(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void)
|
||||
export function post(config: IConfig, url: string, form: any, done: (err: any, result?: string) => void)
|
||||
{
|
||||
authenticate(config, (err) =>
|
||||
{
|
||||
if (err)
|
||||
{
|
||||
return done(err);
|
||||
return done(err);
|
||||
}
|
||||
|
||||
cloudscraper.request(modify(options, 'POST'), (error: Error, response: any, body: any) =>
|
||||
cloudscraper.post(url, getOptions(config, form), (error: Error, response: any, body: any) =>
|
||||
{
|
||||
if (error)
|
||||
{
|
||||
return done(error);
|
||||
return done(error);
|
||||
}
|
||||
done(null, typeof body === 'string' ? body : String(body));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function authUsingCookies(config: IConfig, done: (err: any) => void)
|
||||
{
|
||||
j.setCookie(request.cookie('session_id=' + config.crSessionId + '; Domain=crunchyroll.com; HttpOnly; hostOnly=false;'),
|
||||
CR_COOKIE_DOMAIN);
|
||||
|
||||
checkIfUserIsAuth(config, (errCheckAuth2) =>
|
||||
{
|
||||
if (isAuthenticated)
|
||||
{
|
||||
return done(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
return done(errCheckAuth2);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function authUsingApi(config: IConfig, done: (err: any) => void)
|
||||
{
|
||||
if (!config.pass || !config.user)
|
||||
{
|
||||
log.error('You need to give login/password to use Crunchy');
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
if (config.crDeviceId === undefined)
|
||||
{
|
||||
config.crDeviceId = uuid.v4();
|
||||
}
|
||||
|
||||
if (!config.crSessionUrl || !config.crDeviceType || !config.crAPIVersion ||
|
||||
!config.crLocale || !config.crLoginUrl)
|
||||
{
|
||||
return done(AuthError('Invalid API configuration, please check your config file.'));
|
||||
}
|
||||
|
||||
startSession(config)
|
||||
.then((sessionId: string) =>
|
||||
{
|
||||
// defaultHeaders['Cookie'] = `sess_id=${sessionId}; c_locale=enUS`;
|
||||
return APIlogin(config, sessionId, config.user, config.pass);
|
||||
})
|
||||
.then((userData) =>
|
||||
{
|
||||
checkIfUserIsAuth(config, (errCheckAuth2) =>
|
||||
{
|
||||
if (isAuthenticated)
|
||||
{
|
||||
return done(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
return done(errCheckAuth2);
|
||||
}
|
||||
});
|
||||
})
|
||||
.catch((errInChk) =>
|
||||
{
|
||||
return done(AuthError(errInChk.message));
|
||||
});
|
||||
}
|
||||
|
||||
function authUsingForm(config: IConfig, done: (err: any) => void)
|
||||
{
|
||||
/* So if we are here now, that mean we are not authenticated so do as usual */
|
||||
if (!config.pass || !config.user)
|
||||
{
|
||||
log.error('You need to give login/password to use Crunchy');
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
/* First get https://www.crunchyroll.com/login to get the login token */
|
||||
cloudscraper.get('https://www.crunchyroll.com/login', getOptions(config, null), (err: any, rep: Response, body: string) =>
|
||||
{
|
||||
if (err) return done(err);
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
/* Get the token from the login page */
|
||||
const token = $('input[name="login_form[_token]"]').attr('value');
|
||||
if (token === '')
|
||||
{
|
||||
return done(AuthError('Can\'t find token!'));
|
||||
}
|
||||
|
||||
/* Now call the page again with the token and credentials */
|
||||
const paramForm =
|
||||
{
|
||||
'login_form[name]': config.user,
|
||||
'login_form[password]': config.pass,
|
||||
'login_form[redirect_url]': '/',
|
||||
'login_form[_token]': token
|
||||
};
|
||||
|
||||
cloudscraper.post('https://www.crunchyroll.com/login', getOptions(config, paramForm), (err: any, rep: Response, body: string) =>
|
||||
{
|
||||
if (err)
|
||||
{
|
||||
return done(err);
|
||||
}
|
||||
|
||||
/* Now let's check if we are authentificated */
|
||||
checkIfUserIsAuth(config, (errCheckAuth2) =>
|
||||
{
|
||||
if (isAuthenticated)
|
||||
{
|
||||
return done(null);
|
||||
}
|
||||
else
|
||||
{
|
||||
return done(errCheckAuth2);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Authenticates using the configured pass and user.
|
||||
*/
|
||||
function authenticate(config: IConfig, done: (err: Error) => void)
|
||||
function authenticate(config: IConfig, done: (err: any) => void)
|
||||
{
|
||||
if (isAuthenticated)
|
||||
{
|
||||
return done(null);
|
||||
}
|
||||
|
||||
if (!config.pass || !config.user)
|
||||
/* First of all, check if the user is not already logged via the cookies */
|
||||
checkIfUserIsAuth(config, (errCheckAuth) =>
|
||||
{
|
||||
log.error('You need to give login/password to use Crunchy');
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
startSession()
|
||||
.then((sessionId: string) =>
|
||||
{
|
||||
defaultHeaders.Cookie = `sess_id=${sessionId}; c_locale=enUS`;
|
||||
return login(sessionId, config.user, config.pass);
|
||||
})
|
||||
.then((userData) =>
|
||||
{
|
||||
/**
|
||||
* The page return with a meta based redirection, as we wan't to check that everything is fine, reload
|
||||
* the main page. A bit convoluted, but more sure.
|
||||
*/
|
||||
const options =
|
||||
if (isAuthenticated)
|
||||
{
|
||||
headers: defaultHeaders,
|
||||
jar: true,
|
||||
url: 'http://www.crunchyroll.com/',
|
||||
method: 'GET',
|
||||
};
|
||||
return done(null);
|
||||
}
|
||||
|
||||
cloudscraper.request(options, (err: Error, rep: string, body: string) =>
|
||||
log.info('Seems we are not currently logged. Let\'s login!');
|
||||
|
||||
if (config.logUsingApi)
|
||||
{
|
||||
if (err)
|
||||
{
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
/* Check if auth worked */
|
||||
const regexps = /ga\('set', 'dimension[5-8]', '([^']*)'\);/g;
|
||||
const dims = regexps.exec($('script').text());
|
||||
|
||||
for (let i = 1; i < 5; i++)
|
||||
{
|
||||
if ((dims[i] !== undefined) && (dims[i] !== '') && (dims[i] !== 'not-registered'))
|
||||
{
|
||||
isAuthenticated = true;
|
||||
}
|
||||
|
||||
if ((dims[i] === 'premium') || (dims[i] === 'premiumplus'))
|
||||
{
|
||||
isPremium = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (isAuthenticated === false)
|
||||
{
|
||||
const error = $('ul.message, li.error').text();
|
||||
log.error('Authentication failed: ' + error);
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
if (isPremium === false)
|
||||
{
|
||||
log.warn('Do not use this app without a premium account.');
|
||||
}
|
||||
else
|
||||
{
|
||||
log.info('You have a premium account! Good!');
|
||||
}
|
||||
done(null);
|
||||
});
|
||||
})
|
||||
.catch(done);
|
||||
return authUsingApi(config, done);
|
||||
}
|
||||
else if (config.logUsingCookie)
|
||||
{
|
||||
return authUsingCookies(config, done);
|
||||
}
|
||||
else
|
||||
{
|
||||
return authUsingForm(config, done);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Modifies the options to use the authenticated cookie jar.
|
||||
*/
|
||||
function modify(options: string|request.Options, reqMethod: string): request.Options
|
||||
function getOptions(config: IConfig, form: any)
|
||||
{
|
||||
if (typeof options !== 'string')
|
||||
if (!optionsSet)
|
||||
{
|
||||
options.jar = true;
|
||||
options.headers = defaultHeaders;
|
||||
options.method = reqMethod;
|
||||
return options;
|
||||
currentOptions = {};
|
||||
currentOptions.headers = {};
|
||||
|
||||
currentOptions.headers['Cache-Control'] = 'private';
|
||||
currentOptions.headers.Accept = 'application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5';
|
||||
|
||||
if (config.userAgent)
|
||||
{
|
||||
currentOptions.headers['User-Agent'] = config.userAgent;
|
||||
}
|
||||
else
|
||||
{
|
||||
currentOptions.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0';
|
||||
}
|
||||
|
||||
if (j === undefined)
|
||||
{
|
||||
loadCookies(config);
|
||||
}
|
||||
|
||||
currentOptions.decodeEmails = true;
|
||||
currentOptions.jar = j;
|
||||
optionsSet = true;
|
||||
}
|
||||
return {
|
||||
jar: true,
|
||||
headers: defaultHeaders,
|
||||
url: options.toString(),
|
||||
method: reqMethod
|
||||
};
|
||||
|
||||
currentOptions.form = {};
|
||||
|
||||
if (form !== null)
|
||||
{
|
||||
currentOptions.form = form;
|
||||
}
|
||||
|
||||
|
||||
return currentOptions;
|
||||
}
|
||||
@@ -1,12 +1,13 @@
|
||||
'use strict';
|
||||
import cheerio = require('cheerio');
|
||||
import episode from './episode';
|
||||
// import fs = require('fs');
|
||||
import fs = require('fs-extra');
|
||||
import my_request = require('./my_request');
|
||||
import path = require('path');
|
||||
import url = require('url');
|
||||
import log = require('./log');
|
||||
import languages = require('./languages');
|
||||
|
||||
const persistent = '.crpersistent';
|
||||
|
||||
/**
|
||||
@@ -27,7 +28,7 @@ function fileExist(path: string)
|
||||
/**
|
||||
* Streams the series to disk.
|
||||
*/
|
||||
export default function(config: IConfig, task: IConfigTask, done: (err: Error) => void)
|
||||
export default function(config: IConfig, task: IConfigTask, done: (err: any) => void)
|
||||
{
|
||||
const persistentPath = path.join(config.output || process.cwd(), persistent);
|
||||
|
||||
@@ -45,6 +46,15 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
|
||||
{
|
||||
if (errP)
|
||||
{
|
||||
const reqErr = errP.error;
|
||||
if ((reqErr !== undefined) && (reqErr.syscall))
|
||||
{
|
||||
if ((reqErr.syscall === 'getaddrinfo') && (reqErr.errno === 'ENOTFOUND'))
|
||||
{
|
||||
log.error('The URL \'' + task.address + '\' is invalid, please check => I\'m ignoring it.');
|
||||
}
|
||||
}
|
||||
|
||||
return done(errP);
|
||||
}
|
||||
|
||||
@@ -61,9 +71,20 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
|
||||
{
|
||||
if (errD)
|
||||
{
|
||||
/* Check if domain is valid */
|
||||
const reqErr = errD.error;
|
||||
if ((reqErr !== undefined) && (reqErr.syscall))
|
||||
{
|
||||
if ((reqErr.syscall === 'getaddrinfo') && (reqErr.errno === 'ENOTFOUND'))
|
||||
{
|
||||
page.episodes[i].retry = 0;
|
||||
log.error('The URL \'' + task.address + '\' is invalid, please check => I\'m ignoring it.');
|
||||
}
|
||||
}
|
||||
|
||||
if (page.episodes[i].retry <= 0)
|
||||
{
|
||||
log.error(errD);
|
||||
log.error(JSON.stringify(errD));
|
||||
log.error('Cannot fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
|
||||
'", please rerun later');
|
||||
/* Go to the next on the list */
|
||||
@@ -76,7 +97,7 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
|
||||
if (config.debug)
|
||||
{
|
||||
log.dumpToDebug('series address', task.address);
|
||||
log.dumpToDebug('series error', errD.stack || errD);
|
||||
log.dumpToDebug('series error', JSON.stringify(errD));
|
||||
log.dumpToDebug('series data', JSON.stringify(page));
|
||||
}
|
||||
log.error(errD);
|
||||
@@ -85,7 +106,8 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
|
||||
'" - Retry ' + page.episodes[i].retry + ' / ' + config.retry);
|
||||
page.episodes[i].retry -= 1;
|
||||
}
|
||||
next();
|
||||
setTimeout(next, config.sleepTime);
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -100,13 +122,15 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
|
||||
}
|
||||
|
||||
i += 1;
|
||||
next();
|
||||
setTimeout(next, config.sleepTime);
|
||||
return;
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
i += 1;
|
||||
next();
|
||||
setTimeout(next, config.sleepTime);
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -120,12 +144,13 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
|
||||
*/
|
||||
function download(cache: {[address: string]: number}, config: IConfig,
|
||||
task: IConfigTask, item: ISeriesEpisode,
|
||||
done: (err: Error, ign: boolean) => void)
|
||||
done: (err: any, ign: boolean) => void)
|
||||
{
|
||||
const episodeNumber = parseInt(item.episode, 10);
|
||||
const seasonNumber = item.volume;
|
||||
|
||||
if ( (episodeNumber < task.episode_min) ||
|
||||
(episodeNumber > task.episode_max) )
|
||||
if ( (episodeNumber < task.episode_min.episode) ||
|
||||
(episodeNumber > task.episode_max.episode) )
|
||||
{
|
||||
return done(null, false);
|
||||
}
|
||||
@@ -152,7 +177,7 @@ function download(cache: {[address: string]: number}, config: IConfig,
|
||||
/**
|
||||
* Requests the page and scrapes the episodes and series.
|
||||
*/
|
||||
function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, result?: ISeries) => void)
|
||||
function pageScrape(config: IConfig, task: IConfigTask, done: (err: any, result?: ISeries) => void)
|
||||
{
|
||||
if (task.address[0] === '@')
|
||||
{
|
||||
@@ -161,6 +186,7 @@ function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, resul
|
||||
episodes.push({
|
||||
address: task.address.substr(1),
|
||||
episode: '',
|
||||
seasonName: '',
|
||||
volume: 0,
|
||||
retry: config.retry,
|
||||
});
|
||||
@@ -170,12 +196,13 @@ function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, resul
|
||||
{
|
||||
let episodeCount = 0;
|
||||
my_request.get(config, task.address, (err, result) => {
|
||||
if (err) {
|
||||
if (err)
|
||||
{
|
||||
return done(err);
|
||||
}
|
||||
|
||||
const $ = cheerio.load(result);
|
||||
const title = $('span[itemprop=name]').text();
|
||||
const title = $('meta[itemprop=name]').attr('content');
|
||||
|
||||
if (config.debug)
|
||||
{
|
||||
@@ -193,27 +220,50 @@ function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, resul
|
||||
log.info('Checking availability for ' + title);
|
||||
const episodes: ISeriesEpisode[] = [];
|
||||
|
||||
if ($('.availability-notes-low').length)
|
||||
{
|
||||
log.warn('This serie may have georestriction and some missings episode (like some dubs)' +
|
||||
' [Message: ' + $('.availability-notes-low').text() + '].');
|
||||
}
|
||||
|
||||
if ($('.availability-notes-high').length)
|
||||
{
|
||||
log.warnMore('This serie probably have georestriction and will miss some episodes' +
|
||||
' [Message: ' + $('.availability-notes-high').text() + '].');
|
||||
}
|
||||
|
||||
$('.episode').each((i, el) => {
|
||||
if ($(el).children('img[src*=coming_soon]').length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const season_name = $(el).closest('ul').prev('a').text();
|
||||
const volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
|
||||
const regexp = /Episode\s+((PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i;
|
||||
const regexp = languages.get_epregexp(config);
|
||||
const episode = regexp.exec($(el).children('.series-title').text());
|
||||
const url = $(el).attr('href');
|
||||
|
||||
if ((!url) || (!episode)) {
|
||||
const igndub_re = languages.get_diregexp(config);
|
||||
|
||||
if (config.ignoredub && (igndub_re.exec(season_name)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if ((!url) || (!episode))
|
||||
{
|
||||
return;
|
||||
}
|
||||
episodeCount += 1;
|
||||
episodes.push({
|
||||
address: url,
|
||||
episode: episode[1],
|
||||
seasonName: season_name,
|
||||
volume: volume ? parseInt(volume[0], 10) : 1,
|
||||
retry: config.retry,
|
||||
});
|
||||
});
|
||||
|
||||
if (episodeCount === 0)
|
||||
{
|
||||
log.warn('No episodes found for ' + title + '. Could it be a movie?');
|
||||
|
||||
@@ -9,21 +9,14 @@ import subtitle from '../subtitle/index';
|
||||
/**
|
||||
* Merges the subtitle and video files into a Matroska Multimedia Container.
|
||||
*/
|
||||
export default function(config: IConfig, isSubtitled: boolean, rtmpInputPath: string, filePath: string,
|
||||
streamMode: string, verbose: boolean, done: (err: Error) => void)
|
||||
export default function(config: IConfig, isSubtitled: boolean, videoFileExtention: string, filePath: string,
|
||||
verbose: boolean, done: (err: Error) => void)
|
||||
{
|
||||
const subtitlePath = filePath + '.' + (subtitle.formats[config.format] ? config.format : 'ass');
|
||||
let videoPath = filePath;
|
||||
let cp;
|
||||
|
||||
if (streamMode === 'RTMP')
|
||||
{
|
||||
videoPath += path.extname(rtmpInputPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
videoPath += '.mp4';
|
||||
}
|
||||
videoPath += videoFileExtention;
|
||||
|
||||
cp = childProcess.exec(command() + ' ' +
|
||||
'-o "' + filePath + '.mkv" ' +
|
||||
|
||||
@@ -3,6 +3,7 @@ import childProcess = require('child_process');
|
||||
import os = require('os');
|
||||
import path = require('path');
|
||||
|
||||
import my_request = require('../my_request');
|
||||
import log = require('../log');
|
||||
|
||||
/**
|
||||
@@ -24,7 +25,8 @@ export default function(rtmpUrl: string, rtmpInputPath: string, swfUrl: string,
|
||||
else if (mode === 'HLS')
|
||||
{
|
||||
cmd = command('ffmpeg') + ' ' +
|
||||
'-y -xerror ' +
|
||||
'-user_agent "' + my_request.getUserAgent() + '" ' +
|
||||
'-y -xerror -discard none ' +
|
||||
'-i "' + rtmpInputPath + '" ' +
|
||||
'-c copy -bsf:a aac_adtstoasc ' +
|
||||
'"' + filePath + '.mp4"';
|
||||
|
||||
70
src/vlos.ts
Normal file
70
src/vlos.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
'use strict';
|
||||
|
||||
export default {getMedia};
|
||||
|
||||
function getMedia(vlosScript: string, seasonTitle: string, seasonNumber: string): IEpisodePage
|
||||
{
|
||||
let vlosMedia: IVlosScript;
|
||||
|
||||
function f(script: string) {
|
||||
/* We need to scope things */
|
||||
|
||||
/* This is what will give us the medias */
|
||||
function VilosPlayer() {
|
||||
this.load = function(a: string, b: any, c: any)
|
||||
{
|
||||
vlosMedia = this.config.media;
|
||||
vlosMedia.series = this.config.analytics.media_reporting_parent;
|
||||
};
|
||||
this.config = {};
|
||||
this.config.player = {};
|
||||
this.config.player.pause_screen = {};
|
||||
this.config.language = '';
|
||||
}
|
||||
|
||||
/* Let's stub what the script need */
|
||||
const window = {
|
||||
WM: {
|
||||
UserConsent: {
|
||||
getUserConsentAdvertisingState(): string { return ''; }
|
||||
}
|
||||
}
|
||||
};
|
||||
const document = {
|
||||
getElementsByClassName(a: any): any { return {length: 0}; },
|
||||
};
|
||||
const localStorage = {
|
||||
getItem(a: any): any { return null; },
|
||||
};
|
||||
const $ = {
|
||||
cookie(a: any) { /* nothing */ },
|
||||
};
|
||||
|
||||
/*
|
||||
Evil ugly things. Need to run the script from a somewhat untrusted source.
|
||||
Need to find a better way of doing.
|
||||
*/
|
||||
// tslint:disable-next-line:no-eval
|
||||
eval(script);
|
||||
|
||||
}
|
||||
f(vlosScript);
|
||||
|
||||
if (vlosMedia === undefined)
|
||||
{
|
||||
console.error('Error fetching vlos data - aborting - Please report the error if happen again.');
|
||||
process.exit(-1);
|
||||
}
|
||||
|
||||
return {
|
||||
episode: vlosMedia.metadata.episode_number,
|
||||
id: vlosMedia.metadata.id,
|
||||
series: vlosMedia.series.title,
|
||||
season: seasonTitle,
|
||||
title: vlosMedia.metadata.title,
|
||||
swf: '',
|
||||
volume: seasonNumber,
|
||||
filename: '',
|
||||
media: vlosMedia,
|
||||
};
|
||||
}
|
||||
96
ts.js
96
ts.js
@@ -1,96 +0,0 @@
|
||||
'use strict';
|
||||
var childProcess = require('child_process');
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
var isTest = process.argv[2] === '--only-test';
|
||||
|
||||
// TODO: This build task should be removed upon release of TypeScript 1.5 with
|
||||
// the support for `tsconfig.json`. Invoking `tsc` from `package.json` will then
|
||||
// read the configuration and compile accordingly. It seems that `TSLint` will,
|
||||
// eventually, support this mechanism too. That prevents the need for any kind
|
||||
// of build task and will run entirely based on instructions from `npm`.
|
||||
//
|
||||
// Reference #1: https://github.com/Microsoft/TypeScript/issues/1667
|
||||
// Reference #2: https://github.com/palantir/tslint/issues/281
|
||||
|
||||
read(function(err, fileNames) {
|
||||
clean(fileNames, function() {
|
||||
var hasLintError = false;
|
||||
compile(fileNames, function(err) {
|
||||
if (err) {
|
||||
console.error(err);
|
||||
return process.exit(1);
|
||||
}
|
||||
lint(fileNames, function(message) {
|
||||
process.stdout.write(message);
|
||||
hasLintError = true;
|
||||
}, function() {
|
||||
process.exit(Number(hasLintError));
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Clean the files.
|
||||
* @param {Array.<string>} filePaths
|
||||
* @param {function()} done
|
||||
*/
|
||||
function clean(filePaths, done) {
|
||||
if (isTest) return done();
|
||||
var i = -1;
|
||||
(function next() {
|
||||
i += 1;
|
||||
if (i >= filePaths.length) return done();
|
||||
var filePath = filePaths[i];
|
||||
if (/\.d\.ts$/.test(filePath)) return next();
|
||||
var mapName = filePath.substring(4, filePath.length - 2) + 'js.map';
|
||||
var mapPath = path.join('dist', mapName);
|
||||
if (fs.existsSync(mapPath)) fs.unlinkSync(mapPath);
|
||||
next();
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile the files.
|
||||
* @param {Array.<string>} filePaths
|
||||
* @param {function(Error)} done
|
||||
*/
|
||||
function compile(filePaths, done) {
|
||||
if (isTest) return done(null);
|
||||
var execPath = path.join(__dirname, 'node_modules/.bin/tsc');
|
||||
var options = '--declaration --module CommonJS --noImplicitAny --outDir dist --target ES5';
|
||||
childProcess.exec([execPath, options].concat(filePaths).join(' '), function(err, stdout) {
|
||||
if (stdout) return done(new Error(stdout));
|
||||
done(null);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lint the files.
|
||||
* @param {Array.<string>} filePaths
|
||||
* @param {function(string)} handler
|
||||
* @param {function()} done
|
||||
*/
|
||||
function lint(filePaths, handler, done) {
|
||||
var i = -1;
|
||||
var execPath = path.join(__dirname, 'node_modules/.bin/tslint');
|
||||
(function next() {
|
||||
i += 1;
|
||||
if (i >= filePaths.length) return done();
|
||||
var filePath = filePaths[i];
|
||||
if (/\.d\.ts$/.test(filePath)) return next();
|
||||
childProcess.exec(execPath + ' -f ' + filePath, function(err, stdout) {
|
||||
if (stdout) handler(stdout);
|
||||
next();
|
||||
});
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the files from the project file.
|
||||
* @param {function(Error, Array.<string>)} done
|
||||
*/
|
||||
function read(done) {
|
||||
done(null, JSON.parse(fs.readFileSync('tsconfig.json', 'utf8')).files);
|
||||
}
|
||||
@@ -21,6 +21,7 @@
|
||||
"no-any": false,
|
||||
"no-arg": true,
|
||||
"no-bitwise": true,
|
||||
"prefer-conditional-expression": false,
|
||||
"space-within-parens": false,
|
||||
"no-object-literal-type-assertion": false,
|
||||
"no-console": [true,
|
||||
|
||||
Reference in New Issue
Block a user