1.5.0

Fix login and only direct links issues.

Still need to understand what is happening.

.gitignore

@@ -1,2 +1,3 @@
 dist/
 node_modules/
+package-lock.json

.travis.yml

@@ -3,6 +3,8 @@ sudo: false
 node_js:
  - 8
  - 9
+ - 10
+ - 11
 before_install:
  - npm install --only=dev
 script:

README.md

@@ -1,6 +1,10 @@
 # Crunchy: a fork of Deathspike/CrunchyRoll.js
 
-[![Issue Stats](http://issuestats.com/github/Godzil/Crunchy/badge/issue)](http://issuestats.com/github/Godzil/Crunchy) [![Travis CI](https://travis-ci.org/Godzil/Crunchy.svg?branch=master)](https://travis-ci.org/Godzil/Crunchy) [![Maintainability](https://api.codeclimate.com/v1/badges/413c7ca11c0805b1ef3e/maintainability)](https://codeclimate.com/github/Godzil/Crunchy/maintainability) 
+[![Travis CI](https://travis-ci.org/Godzil/Crunchy.svg?branch=master)](https://travis-ci.org/Godzil/Crunchy)
+[![Maintainability](https://api.codeclimate.com/v1/badges/413c7ca11c0805b1ef3e/maintainability)](https://codeclimate.com/github/Godzil/Crunchy/maintainability)
+![npm](https://img.shields.io/npm/dy/crunchy.svg)
+![Issue Count](https://img.shields.io/github/issues/Godzil/Crunchy.svg)
+![npm](https://img.shields.io/npm/v/crunchy.svg?label=Last%20published%20version)
 
 *Crunchy* is capable of downloading *anime* episodes from the popular *CrunchyRoll* streaming service. An episode is stored in the original video format (often H.264 in a MP4 container) and the configured subtitle format (ASS or SRT).The two output files are then merged into a single MKV file.
 

package.json

@@ -15,40 +15,41 @@
   "engines": {
     "node": ">=5.0"
   },
-  "version": "1.4.1",
+  "version": "1.5.0",
   "bin": {
     "crunchy": "./bin/crunchy",
     "crunchy.sh": "./bin/crunchy.sh"
   },
   "dependencies": {
-    "big-integer": "^1.6.32",
-    "bluebird": "^3.5.1",
+    "big-integer": "^1.6.44",
+    "bluebird": "^3.5.5",
+    "brotli": "^1.3.2",
     "cheerio": "^0.22.0",
-    "cloudscraper": "^1.5.0",
-    "commander": "^2.16.0",
-    "fs-extra": "^7.0.0",
+    "cloudscraper": "^4.1.2",
+    "commander": "^2.20.0",
+    "fs-extra": "^8.1.0",
     "mkdirp": "^0.5.0",
     "pjson": "^1.0.9",
-    "request": "^2.87.0",
-    "request-promise": "^4.2.2",
+    "request": "^2.88.0",
+    "request-promise": "^4.2.4",
     "tough-cookie-file-store": "^1.2.0",
     "uuid": "^3.3.2",
     "xml2js": "^0.4.5"
   },
   "devDependencies": {
-    "@types/bluebird": "^3.5.23",
-    "@types/cheerio": "^0.22.8",
-    "@types/fs-extra": "^5.0.4",
+    "@types/bluebird": "^3.5.27",
+    "@types/cheerio": "^0.22.12",
+    "@types/fs-extra": "^8.0.0",
     "@types/mkdirp": "^0.5.2",
-    "@types/node": "^10.5.3",
-    "@types/request": "^2.47.1",
-    "@types/request-promise": "^4.1.42",
-    "@types/uuid": "^3.4.3",
-    "@types/xml2js": "^0.4.3",
-    "npm-check": "^5.7.1",
+    "@types/node": "^12.6.8",
+    "@types/request": "^2.48.2",
+    "@types/request-promise": "^4.1.44",
+    "@types/uuid": "^3.4.5",
+    "@types/xml2js": "^0.4.4",
+    "npm-check": "^5.9.0",
     "tsconfig-lint": "^0.12.0",
-    "tslint": "^5.11.0",
-    "typescript": "^2.9.2"
+    "tslint": "^5.18.0",
+    "typescript": "^3.5.3"
   },
   "scripts": {
     "prepublishOnly": "npm run build",

src/batch.ts

@@ -33,6 +33,11 @@ export default function(args: string[], done: (err?: Error) => void)
     batchPath = path.normalize(path.join(process.cwd(), config.batch));
   }
 
+  if (config.nametmpl === undefined)
+  {
+    config.nametmpl = '{SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - {EPISODE_TITLE} - [{TAG}]';
+  }
+
   // Update the config file with new parameters
   cfg.save(config);
 
@@ -88,7 +93,7 @@ export default function(args: string[], done: (err?: Error) => void)
         return done(err);
     }
 
-    if (tasksArr[0].address === '')
+    if (!tasksArr || !tasksArr[0] || (tasksArr[0].address === ''))
     {
         return done();
     }
@@ -252,7 +257,15 @@ function checkURL(address: string): boolean
   {
     return true;
   }
-  if (address.startsWith('http:\/\/'))
+  if (address.startsWith('https:\/\/'))
+  {
+    return true;
+  }
+  if (address.startsWith('@http:\/\/'))
+  {
+    return true;
+  }
+  if (address.startsWith('@https:\/\/'))
   {
     return true;
   }
@@ -341,11 +354,12 @@ function parse(args: string[]): IConfigLine
     // Episode filter
     .option('-e, --episodes <s>', 'Episode list. Read documentation on how to use')
     // Settings
+    .option('-l, --crlang <s>', 'CR page language (valid: en, fr, es, it, pt, de, ru).')
     .option('-f, --format <s>', 'The subtitle format.', 'ass')
     .option('-o, --output <s>', 'The output path.')
     .option('-s, --series <s>', 'The series name override.')
     .option('--ignoredub', 'Experimental: Ignore all seasons where the title end with \'Dub)\'')
-    .option('-n, --nametmpl <s>', 'Output name template', '{SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - {EPISODE_TITLE} - [{TAG}]')
+    .option('-n, --nametmpl <s>', 'Output name template')
     .option('-t, --tag <s>', 'The subgroup.', 'CrunchyRoll')
     .option('-r, --resolution <s>', 'The video resolution. (valid: 360, 480, 720, 1080)', '1080')
     .option('-b, --batch <s>', 'Batch file', 'CrunchyRoll.txt')

src/cli.ts

@@ -11,18 +11,25 @@ log.info('Crunchy version ' + current_version);
 request.get({ uri: 'https://box.godzil.net/getVersion.php?tool=crunchy&v=' + current_version },
                   (error: Error, response: any, body: any) =>
 {
-  const onlinepkg = JSON.parse(body);
-  if (onlinepkg.status === 'ok')
+  if (response && (response.statusCode === 200))
   {
-    let tmp = current_version.split('.');
-    const cur = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
-    tmp = onlinepkg.version.split('.');
-    const dist = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
-    if (dist > cur)
+    const onlinepkg = JSON.parse(body);
+    if (onlinepkg.status === 'ok')
     {
-      log.warnMore('There is a newer version of crunchy (v' + onlinepkg.version + '), you should update!');
+      let tmp = current_version.split('.');
+      const cur = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
+      tmp = onlinepkg.version.split('.');
+      const dist = (Number(tmp[0]) * 10000) + (Number(tmp[1]) * 100) + Number(tmp[2]);
+      if (dist > cur)
+      {
+        log.warnMore('There is a newer version of crunchy (v' + onlinepkg.version + '), you should update!');
+      }
     }
   }
+  else
+  {
+    log.info('Error while checking for the current version.');
+  }
 });
 
 batch(process.argv, (err: any) =>

src/episode.ts

@@ -66,7 +66,7 @@ function fileExist(path: string)
 
 function sanitiseFileName(str: string)
 {
-  return str.replace(/[\/':\?\*"<>\\\.]/g, '_');
+  return str.replace(/[\/':\?\*"<>\\\.\|]/g, '_');
 }
 
 /**

src/interface/IConfig.d.ts

@@ -7,6 +7,7 @@ interface IConfig {
   merge?: boolean;
   episodes?: string;
   // Settings
+  crlang?: string;
   format?: string;
   output?: string;
   series?: string;

src/languages.ts

@@ -0,0 +1,66 @@
+'use strict';
+
+const localeCC: { [id: string]: string; } =
+{
+  enUS: 'en', enGB: 'en',
+  esLA: 'es', esES: 'es',
+  ptPT: 'pt', ptBR: 'pt',
+  frFR: 'fr',
+  deDE: 'de',
+  itIT: 'it',
+  ruRU: 'ru',
+};
+
+export function localeToCC(locale: string): string
+{
+  let ret = localeCC.enGB;
+
+  if (locale in localeCC)
+  {
+    ret = localeCC[locale];
+  }
+
+  return ret;
+}
+
+const dubignore_regexp: { [id: string]: RegExp; } =
+{
+  en: /\(.*Dub(?:bed)?.*\)|(?:\(RU\))/i,
+  fr: /\(.*Dub(?:bed)?.*\)|(?:\(RU\))|\(?Doublage.*\)?/,
+  de: /\(.*isch\)|\(Dubbed\)|\(RU\)/
+};
+
+export function get_diregexp(config: IConfig): RegExp
+{
+  let ret = dubignore_regexp.en;
+
+  if (config.crlang in dubignore_regexp)
+  {
+    ret = dubignore_regexp[config.crlang];
+  }
+
+  return ret;
+}
+
+const episodes_regexp: { [id: string]: RegExp; } =
+{
+  en: /Episode\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+  fr: /Épisode\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+  de: /Folge\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+  es: /Episodio\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+  it: /Episodio\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+  pt: /Episódio\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+  ru: /Серия\s+((OVA)|(PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i,
+};
+
+export function get_epregexp(config: IConfig): RegExp
+{
+  let ret = episodes_regexp.en;
+
+  if (config.crlang in episodes_regexp)
+  {
+    ret = episodes_regexp[config.crlang];
+  }
+
+  return ret;
+}

src/my_request.ts

@@ -6,6 +6,7 @@ import Promise = require('bluebird');
 import uuid = require('uuid');
 import path = require('path');
 import fs = require('fs-extra');
+import languages = require('./languages');
 import log = require('./log');
 
 import { RequestPromise } from 'request-promise';
@@ -13,8 +14,6 @@ import { Response } from 'request';
 
 // tslint:disable-next-line:no-var-requires
 const cookieStore = require('tough-cookie-file-store');
-// tslint:disable-next-line:no-var-requires
-const cloudscraper = require('cloudscraper');
 
 const CR_COOKIE_DOMAIN = 'http://crunchyroll.com';
 
@@ -23,13 +22,27 @@ let isPremium = false;
 
 let j: request.CookieJar;
 
-const defaultHeaders: request.Headers =
+const defaultHeaders =
 {
-  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
+  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
   'Connection': 'keep-alive',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
   'Referer': 'https://www.crunchyroll.com/login',
+  'Cache-Control': 'private',
+  'Accept-Language': 'en-US,en;q=0.9'
 };
 
+const defaultOptions =
+{
+  followAllRedirects: true,
+  decodeEmails: true,
+  challengesToSolve: 3,
+  gzip: true,
+};
+
+// tslint:disable-next-line:no-var-requires
+const cloudscraper = require('cloudscraper').defaults(defaultOptions);
+
 function AuthError(msg: string): IAuthError
 {
   return { name: 'AuthError', message: msg, authError: true };
@@ -62,7 +75,7 @@ function startSession(config: IConfig): Promise<any>
   });
 }
 
-function login(config: IConfig, sessionId: string, user: string, pass: string): Promise<any>
+function APIlogin(config: IConfig, sessionId: string, user: string, pass: string): Promise<any>
 {
   return rp(
   {
@@ -95,14 +108,9 @@ function checkIfUserIsAuth(config: IConfig, done: (err: Error) => void): void
   /**
    * The main page give us some information about the user
    */
-  const options =
-  {
-    headers: defaultHeaders,
-    jar: j,
-    url: 'http://www.crunchyroll.com/',
-    method: 'GET',
-  };
-  cloudscraper.request(options, (err: Error, rep: string, body: string) =>
+  const url = 'http://www.crunchyroll.com/';
+
+  cloudscraper.get({gzip: true, uri: url}, (err: Error, rep: string, body: string) =>
   {
     if (err)
     {
@@ -111,6 +119,22 @@ function checkIfUserIsAuth(config: IConfig, done: (err: Error) => void): void
 
     const $ = cheerio.load(body);
 
+    /* As we are here, try to detect which locale CR tell us */
+    const localeRE = /LOCALE = "([a-zA-Z]+)",/g;
+    const locale = localeRE.exec($('script').text())[1];
+    const countryCode = languages.localeToCC(locale);
+
+    if (config.crlang === undefined)
+    {
+      log.info('No locale set. Setting to the one reported by CR: "' + countryCode + '"');
+      config.crlang = countryCode;
+    }
+    else if (config.crlang !== countryCode)
+    {
+      log.warn('Crunchy is configured for locale "' + config.crlang + '" but CR report "' + countryCode + '" (LOCALE = ' + locale + ')');
+      log.warn('Check if it is correct or rerun (once) with "-l ' + countryCode + '" to correct.');
+    }
+
     /* Check if auth worked */
     const regexps = /ga\('set', 'dimension[5-8]', '([^']*)'\);/g;
     const dims = regexps.exec($('script').text());
@@ -131,6 +155,11 @@ function checkIfUserIsAuth(config: IConfig, done: (err: Error) => void): void
     if (isAuthenticated === false)
     {
         const error = $('ul.message, li.error').text();
+        log.warn('Authentication failed: ' + error);
+
+        log.dumpToDebug('not auth rep', rep);
+        log.dumpToDebug('not auth body', body);
+
         return done(AuthError('Authentication failed: ' + error));
     }
     else
@@ -181,7 +210,7 @@ export function getUserAgent(): string
 /**
  * Performs a GET request for the resource.
  */
-export function get(config: IConfig, options: string|request.Options, done: (err: any, result?: string) => void)
+export function get(config: IConfig, options: string|any, done: (err: any, result?: string) => void)
 {
   if (j === undefined)
   {
@@ -200,7 +229,7 @@ export function get(config: IConfig, options: string|request.Options, done: (err
       return done(err);
     }
 
-    cloudscraper.request(modify(options, 'GET'), (error: any, response: any, body: any) =>
+    cloudscraper.get(modify(options), (error: any, response: any, body: any) =>
     {
       if (error) return done(error);
 
@@ -231,7 +260,7 @@ export function post(config: IConfig, options: request.Options, done: (err: Erro
       return done(err);
     }
 
-    cloudscraper.request(modify(options, 'POST'), (error: Error, response: any, body: any) =>
+    cloudscraper.post(modify(options), (error: Error, response: any, body: any) =>
     {
       if (error)
       {
@@ -261,7 +290,7 @@ function authenticate(config: IConfig, done: (err: Error) => void)
     }
 
     /* So if we are here now, that mean we are not authenticated so do as usual */
-    if (!config.pass || !config.user)
+    if ((!config.logUsingApi && !config.logUsingCookie) && (!config.pass || !config.user))
     {
       log.error('You need to give login/password to use Crunchy');
       process.exit(-1);
@@ -285,8 +314,8 @@ function authenticate(config: IConfig, done: (err: Error) => void)
       startSession(config)
       .then((sessionId: string) =>
       {
-        defaultHeaders.Cookie = `sess_id=${sessionId}; c_locale=enUS`;
-        return login(config, sessionId, config.user, config.pass);
+        // defaultHeaders['Cookie'] = `sess_id=${sessionId}; c_locale=enUS`;
+        return APIlogin(config, sessionId, config.user, config.pass);
       })
       .then((userData) =>
       {
@@ -331,14 +360,11 @@ function authenticate(config: IConfig, done: (err: Error) => void)
       /* First get https://www.crunchyroll.com/login to get the login token */
       const options =
       {
-        headers: defaultHeaders,
-        jar: j,
-        gzip: false,
-        method: 'GET',
-        url: 'https://www.crunchyroll.com/login'
+        // jar: j,
+        uri: 'https://www.crunchyroll.com/login'
       };
 
-      cloudscraper.request(options, (err: Error, rep: string, body: string) =>
+      cloudscraper.get(options, (err: Error, rep: string, body: string) =>
       {
         if (err) return done(err);
 
@@ -354,7 +380,6 @@ function authenticate(config: IConfig, done: (err: Error) => void)
         /* Now call the page again with the token and credentials */
         const options =
         {
-          headers: defaultHeaders,
           form:
           {
             'login_form[name]': config.user,
@@ -362,13 +387,11 @@ function authenticate(config: IConfig, done: (err: Error) => void)
             'login_form[redirect_url]': '/',
             'login_form[_token]': token
           },
-          jar: j,
-          gzip: false,
-          method: 'POST',
+          // jar: j,
           url: 'https://www.crunchyroll.com/login'
         };
 
-        cloudscraper.request(options, (err: Error, rep: string, body: string) =>
+        cloudscraper.post(options, (err: Error, rep: string, body: string) =>
         {
           if (err)
           {
@@ -396,19 +419,15 @@ function authenticate(config: IConfig, done: (err: Error) => void)
 /**
  * Modifies the options to use the authenticated cookie jar.
  */
-function modify(options: string|request.Options, reqMethod: string): request.Options
+function modify(options: string|any): any
 {
   if (typeof options !== 'string')
   {
     options.jar = j;
-    options.headers = defaultHeaders;
-    options.method = reqMethod;
     return options;
   }
   return {
     jar: j,
-    headers: defaultHeaders,
     url: options.toString(),
-    method: reqMethod
   };
 }

src/series.ts

@@ -1,12 +1,13 @@
 'use strict';
 import cheerio = require('cheerio');
 import episode from './episode';
-// import fs = require('fs');
 import fs = require('fs-extra');
 import my_request = require('./my_request');
 import path = require('path');
 import url = require('url');
 import log  = require('./log');
+import languages = require('./languages');
+
 const persistent = '.crpersistent';
 
 /**
@@ -197,7 +198,7 @@ function pageScrape(config: IConfig, task: IConfigTask, done: (err: any, result?
       }
 
       const $ = cheerio.load(result);
-      const title = $('span[itemprop=name]').text();
+      const title = $('meta[itemprop=name]').attr('content');
 
       if (config.debug)
       {
@@ -234,11 +235,13 @@ function pageScrape(config: IConfig, task: IConfigTask, done: (err: any, result?
 
         const season_name = $(el).closest('ul').prev('a').text();
         const volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
-        const regexp = /Episode\s+((PV )?[S0-9][\-P0-9.]*[a-fA-F]?)\s*$/i;
+        const regexp = languages.get_epregexp(config);
         const episode = regexp.exec($(el).children('.series-title').text());
         const url = $(el).attr('href');
 
-        if (config.ignoredub && (season_name.endsWith('Dub)') || season_name.endsWith('dub)')))
+        const igndub_re = languages.get_diregexp(config);
+
+        if (config.ignoredub && (igndub_re.exec(season_name)))
         {
           return;
         }

ts.js

@@ -1,96 +0,0 @@
-'use strict';
-var childProcess = require('child_process');
-var fs = require('fs');
-var path = require('path');
-var isTest = process.argv[2] === '--only-test';
-
-// TODO: This build task should be removed upon release of TypeScript 1.5 with
-// the support for `tsconfig.json`. Invoking `tsc` from `package.json` will then
-// read the configuration and compile accordingly. It seems that `TSLint` will,
-// eventually, support this mechanism too. That prevents the need for any kind
-// of build task and will run entirely based on instructions from `npm`.
-//
-// Reference #1: https://github.com/Microsoft/TypeScript/issues/1667
-// Reference #2: https://github.com/palantir/tslint/issues/281
-
-read(function(err, fileNames) {
-  clean(fileNames, function() {
-    var hasLintError = false;
-    compile(fileNames, function(err) {
-      if (err) {
-        console.error(err);
-        return process.exit(1);
-      }
-      lint(fileNames, function(message) {
-        process.stdout.write(message);
-        hasLintError = true;
-      }, function() {
-        process.exit(Number(hasLintError));
-      });
-    });
-  });
-});
-
-/**
- * Clean the files.
- * @param {Array.<string>} filePaths
- * @param {function()} done
- */
-function clean(filePaths, done) {
-  if (isTest) return done();
-  var i = -1;
-  (function next() {
-    i += 1;
-    if (i >= filePaths.length) return done();
-    var filePath = filePaths[i];
-    if (/\.d\.ts$/.test(filePath)) return next();
-    var mapName = filePath.substring(4, filePath.length - 2) + 'js.map';
-    var mapPath = path.join('dist', mapName);
-    if (fs.existsSync(mapPath)) fs.unlinkSync(mapPath);
-    next();
-  })();
-}
-
-/**
- * Compile the files.
- * @param {Array.<string>} filePaths
- * @param {function(Error)} done
- */
-function compile(filePaths, done) {
-  if (isTest) return done(null);
-  var execPath = path.join(__dirname, 'node_modules/.bin/tsc');
-  var options = '--declaration --module CommonJS --noImplicitAny --outDir dist --target ES5';
-  childProcess.exec([execPath, options].concat(filePaths).join(' '), function(err, stdout) {
-    if (stdout) return done(new Error(stdout));
-    done(null);
-  });
-}
-
-/**
- * Lint the files.
- * @param {Array.<string>} filePaths
- * @param {function(string)} handler
- * @param {function()} done
- */
-function lint(filePaths, handler, done) {
-  var i = -1;
-  var execPath = path.join(__dirname, 'node_modules/.bin/tslint');
-  (function next() {
-    i += 1;
-    if (i >= filePaths.length) return done();
-    var filePath = filePaths[i];
-    if (/\.d\.ts$/.test(filePath)) return next();
-    childProcess.exec(execPath + ' -f ' + filePath, function(err, stdout) {
-      if (stdout) handler(stdout);
-      next();
-    });
-  })();
-}
-
-/**
- * Read the files from the project file.
- * @param {function(Error, Array.<string>)} done
- */
-function read(done) {
-  done(null, JSON.parse(fs.readFileSync('tsconfig.json', 'utf8')).files);
-}