Fix login issue

- Use the cloudscraper layer on top of request to pass through the cloudfare browser check
- switch from tsd to typings

README.md

@@ -1,6 +1,6 @@
-# Crunchy.js: a fork of Deathspike/CrunchyRoll.js
+# Crunchy: a fork of Deathspike/CrunchyRoll.js
 
-*Crunchy.js* is capable of downloading *anime* episodes from the popular *CrunchyRoll* streaming service. An episode is stored in the original video format (often H.264 in a MP4 container) and the configured subtitle format (ASS or SRT).The two output files are then merged into a single MKV file.
+*Crunchy* is capable of downloading *anime* episodes from the popular *CrunchyRoll* streaming service. An episode is stored in the original video format (often H.264 in a MP4 container) and the configured subtitle format (ASS or SRT).The two output files are then merged into a single MKV file.
 
 ## Motivation
 
@@ -10,7 +10,7 @@
 
 This application is not endorsed or affliated with *CrunchyRoll*. The usage of this application enables episodes to be downloaded for offline convenience which may be forbidden by law in your country. Usage of this application may also cause a violation of the agreed *Terms of Service* between you and the stream provider. A tool is not responsible for your actions; please make an informed decision prior to using this application.
 
-**PLEASE USE THIS TOOL ONLY IF YOU HAVE A PREMIUM ACCOUNT**
+**PLEASE _ONLY_ USE THIS TOOL IF YOU HAVE A _PREMIUM ACCOUNT_**
 
 ## Configuration
 

package.json

@@ -1,6 +1,6 @@
 {
   "author": "Godzil",
-  "description": "Crunchy.js is a fork of Crunchyroll.js, capable of downloading anime episodes from the popular CrunchyRoll streaming service.",
+  "description": "Crunchy is a fork of Crunchyroll.js, capable of downloading anime episodes from the popular CrunchyRoll streaming service.",
   "license": "MIT",
   "keywords": [
     "anime",
@@ -12,26 +12,27 @@
     "type": "git",
     "url": "git://github.com/Godzil/crunchyroll.js.git"
   },
-  "version": "1.1.9",
+  "version": "1.1.15",
   "bin": {
     "crunchy": "./bin/crunchy"
   },
   "dependencies": {
     "big-integer": "1.4.4",
-    "cheerio": "0.18.0",
+    "cheerio": "0.22.0",
+    "cloudscraper": "1.4.1",
     "commander": "2.6.0",
     "mkdirp": "0.5.0",
-    "request": "2.53.0",
+    "request": "2.74.0",
     "xml2js": "0.4.5"
   },
   "devDependencies": {
-    "tsd": "0.5.7",
+    "typing": "2.1.0",
     "tslint": "2.3.0-beta",
     "typescript": "1.5.0-beta"
   },
   "scripts": {
-    "prepublish": "npm run tsd && tsc",
+    "prepublish": "npm run types && tsc",
     "test": "node ts --only-test",
-    "tsd": "tsd reinstall -o -s"
+    "types": "typings install"
   }
 }

src/episode.ts

@@ -7,6 +7,7 @@ import path = require('path');
 import subtitle from './subtitle/index';
 import video from './video/index';
 import xml2js = require('xml2js');
+import log  = require('./log');
 
 /**
  * Streams the episode to disk.
@@ -24,12 +25,12 @@ export default function(config: IConfig, address: string, done: (err: Error, ign
 /**
  * Completes a download and writes the message with an elapsed time.
  */
-function complete(message: string, begin: number, done: (err: Error, ign: boolean) => void) {
+function complete(epName: string, message: string, begin: number, done: (err: Error, ign: boolean) => void) {
   var timeInMs = Date.now() - begin;
   var seconds = prefix(Math.floor(timeInMs / 1000) % 60, 2);
   var minutes = prefix(Math.floor(timeInMs / 1000 / 60) % 60, 2);
   var hours = prefix(Math.floor(timeInMs / 1000 / 60 / 60), 2);
-  console.log(message + ' (' + hours + ':' + minutes + ':' + seconds + ')');
+  log.dispEpisode(epName, message + ' (' + hours + ':' + minutes + ':' + seconds + ')', true);
   done(null, false);
 }
 
@@ -51,20 +52,20 @@ function fileExist(path: string) {
  */
 function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void) {
   var series = config.series || page.series;
-  series = series.replace("/","_").replace("'","_");
-  var fileName = name(config, page, series, "").replace("/","_").replace("'","_");
+  series = series.replace("/","_").replace("'","_").replace(":","_");
+  var fileName = name(config, page, series, "").replace("/","_").replace("'","_").replace(":","_");
   var filePath = path.join(config.output || process.cwd(), series, fileName);
   if (fileExist(filePath + ".mkv"))
   {
     var count = 0;
-    console.info("File '"+fileName+"' already exist...");
+    log.warn("File '"+fileName+"' already exist...");
     do
     {
       count = count + 1;
-      fileName = name(config, page, series, "-" + count).replace("/","_").replace("'","_");
+      fileName = name(config, page, series, "-" + count).replace("/","_").replace("'","_").replace(":","_");
       filePath = path.join(config.output || process.cwd(), series, fileName);
     } while(fileExist(filePath + ".mkv"))
-    console.info("Renaming to '"+fileName+"'...");
+   log.warn("Renaming to '"+fileName+"'...");
   }
 
   mkdirp(path.dirname(filePath), (err: Error) => {
@@ -74,20 +75,20 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
       var now = Date.now();
 			if (player.video.file != undefined)
 			{
-        console.log('Fetching ' + fileName);
+        log.dispEpisode(fileName, 'Fetching...', false);
         downloadVideo(config, page, player, filePath, err => {
           if (err) return done(err, false);
-          if (config.merge) return complete('Finished ' + fileName, now, done);
+          if (config.merge) return complete(fileName, 'Finished!', now, done);
           var isSubtited = Boolean(player.subtitle);
           video.merge(config, isSubtited, player.video.file, filePath, player.video.mode, err => {
             if (err) return done(err, false);
-            complete('Finished ' + fileName, now, done);
+            complete(fileName, 'Finished!', now, done);
           });
         });
       }
       else
       {
-        console.log('Ignoring ' + fileName + ': not released yet');
+        log.dispEpisode(fileName, 'Ignoring: not released yet', true);
         done(null, true);
       }
     });
@@ -137,7 +138,7 @@ function name(config: IConfig, page: IEpisodePage, series: string, extra: string
   var episode = (episodeNum < 10 ? '0' : '') + page.episode;
   var volume = (volumeNum < 10 ? '0' : '') + page.volume;
   var tag = config.tag || 'CrunchyRoll';
-  return series + ' ' + volume + 'x' + episode + extra + ' [' + tag + ']';  
+  return series + ' - s' + volume + 'e' + episode +' - [' + tag + ']' + extra;  
 }
 
 /**
@@ -166,8 +167,8 @@ function scrapePage(config: IConfig, address: string, done: (err: Error, page?:
 
     if (!swf || !data)
     {
-      console.info('Something wrong in the page at '+address+' (data are: '+look+')');
-      console.info('Setting Season to 0 and episode to \’0\’...');
+      log.warn('Something wrong in the page at '+address+' (data are: '+look+')');
+      log.warn('Setting Season to 0 and episode to \’0\’...');
       done(null, {
           id: id,
           episode: "0",

src/log.ts

@@ -0,0 +1,71 @@
+'use strict';
+import os = require('os');
+
+export function error(str: string)
+{
+  if (os.platform() === 'win32')
+  {
+    console.log(' * ERROR: ' + str);
+  }
+  else
+  {
+    /* Do more fancy output */
+    console.error(' \x1B[1;31m* ERROR\x1B[0m: ' + str);
+  }
+}
+
+export function info(str: string)
+{
+  if (os.platform() === 'win32')
+  {
+    console.log(' * INFO : ' + str);
+  }
+  else
+  {
+    /* Do more fancy output */
+    console.log(' \x1B[1;32m* INFO \x1B[0m: ' + str);
+  }
+}
+
+export function debug(str: string)
+{
+  if (os.platform() === 'win32')
+  {
+    console.log(' * DEBUG: ' + str);
+  }
+  else
+  {
+    /* Do more fancy output */
+    console.log(' \x1B[1;35m* DEBUG\x1B[0m: ' + str);
+  }
+}
+
+export function warn(str: string)
+{
+  if (os.platform() === 'win32')
+  {
+    console.log(' * WARN : ' + str);
+  }
+  else
+  {
+    /* Do more fancy output */
+    console.log(' \x1B[1;33m* WARN \x1B[0m: ' + str);
+  }
+}
+
+export function dispEpisode(name: string, status: string, addNL: boolean)
+{
+  if (os.platform() === 'win32')
+  {
+    process.stdout.write(' > ' + name + ' : ' + status + '\x1B[0G');
+  }
+  else
+  {
+    /* Do more fancy output */
+    process.stdout.write(' \x1B[1;33m> \x1B[37m' + name + '\x1B[0m : \x1B[33m' + status + '\x1B[0m\x1B[0G');
+  }
+  if (addNL)
+  {
+    console.log('');
+  }
+}

src/request.ts

@@ -1,6 +1,15 @@
 'use strict';
 import request = require('request');
+import cheerio = require('cheerio');
+import log = require('./log');
+var cloudscraper = require('cloudscraper');
 var isAuthenticated = false;
+var isPremium = false;
+
+var defaultHeaders: request.Headers = {
+  'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+  'Connection': 'keep-alive'
+};
 
 /**
  * Performs a GET request for the resource.
@@ -8,7 +17,7 @@ var isAuthenticated = false;
 export function get(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void) {
   authenticate(config, err => {
     if (err) return done(err);
-    request.get(modify(options), (err: Error, response: any, body: any) => {
+    cloudscraper.request(modify(options, 'GET'), (err: Error, response: any, body: any) => {
       if (err) return done(err);
       done(null, typeof body === 'string' ? body : String(body));
     });
@@ -21,7 +30,7 @@ export function get(config: IConfig, options: request.Options, done: (err: Error
 export function post(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void) {
   authenticate(config, err => {
     if (err) return done(err);
-    request.post(modify(options), (err: Error, response: any, body: any) => {
+    cloudscraper.request(modify(options, 'POST'), (err: Error, response: any, body: any) => {
       if (err) return done(err);
       done(null, typeof body === 'string' ? body : String(body));
     });
@@ -33,30 +42,92 @@ export function post(config: IConfig, options: request.Options, done: (err: Erro
  */
 function authenticate(config: IConfig, done: (err: Error) => void) {
   if (isAuthenticated || !config.pass || !config.user) return done(null);
-  var options = {
-    form: {
-      formname: 'RpcApiUser_Login',
-      fail_url: 'https://www.crunchyroll.com/login',
-      name: config.user,
-      password: config.pass
-    },
+
+  /* Bypass the login page and send a login request directly */
+  var options =
+  {
+    headers: defaultHeaders,
     jar: true,
-    url: 'https://www.crunchyroll.com/?a=formhandler'
+    gzip: false,
+    method: 'GET',
+    url: 'https://www.crunchyroll.com/login'
   };
-  request.post(options, (err: Error) => {
+
+  // request(options, (err: Error, rep: string, body: string) =>
+  cloudscraper.request(options, (err: Error, rep: string, body: string) =>
+  {
     if (err) return done(err);
-    isAuthenticated = true;
-    done(null);
+
+    var $ = cheerio.load(body);
+
+    /* Get the token from the login page */
+    var token = $('input[name="login_form[_token]"]').attr('value');
+    if (token === '') return done(new Error('Can`t find token!'));
+
+    var options =
+    {
+      headers: defaultHeaders,
+      form:
+      {
+        'login_form[redirect_url]': '/',
+        'login_form[name]': config.user,
+        'login_form[password]': config.pass,
+        'login_form[_token]': token
+      },
+      jar: true,
+      gzip: false,
+      method: 'POST',
+      url: 'https://www.crunchyroll.com/login'
+    };
+    // request.post(options, (err: Error, rep: string, body: string) =>
+    cloudscraper.request(options, (err: Error, rep: string, body: string) =>
+    {
+      if (err) return done(err);
+      /* The page return with a meta based redirection, as we wan't to check that everything is fine, reload
+       * the main page. A bit convoluted, but more sure.
+       */
+      var options =
+      {
+        headers: defaultHeaders,
+        jar: true,
+        url: 'http://www.crunchyroll.com/',
+        method: 'GET'
+      };
+
+      cloudscraper.request(options, (err: Error, rep: string, body: string) =>
+      {
+        if (err) return done(err);
+        var $ = cheerio.load(body);
+        /* Check if auth worked */
+        var regexps = /ga\(\'set\', \'dimension[5-8]\', \'([^']*)\'\);/g;
+        var dims = regexps.exec($('script').text());
+        for (var i = 1; i < 5; i++)
+        {
+          if ((dims[i] !== undefined) && (dims[i] !== '') && (dims[i] !== 'not-registered')) { isAuthenticated = true; }
+          if ((dims[i] === 'premium') || (dims[i] === 'premiumplus')) { isPremium = true; }
+        }
+        if (isAuthenticated === false)
+        {
+          var error = $('ul.message, li.error').text();
+          return done(new Error('Authentication failed: ' + error));
+        }
+        if (isPremium === false) { log.warn('Do not use this app without a premium account.'); }
+        else { log.info('You have a premium account! Good!'); }
+        done(null);
+      });
+    });
   });
 }
 
 /**
  * Modifies the options to use the authenticated cookie jar.
  */
-function modify(options: string|request.Options): request.Options {
+function modify(options: string|request.Options, reqMethod: string): request.Options {
   if (typeof options !== 'string') {
     options.jar = true;
+    options.headers = defaultHeaders;
+    options.method = reqMethod;
     return options;
   }
-  return {jar: true, url: options.toString()};
+  return { jar: true, headers: defaultHeaders, url: options.toString(), method: reqMethod};
 }

src/series.ts

@@ -5,6 +5,7 @@ import fs = require('fs');
 import request = require('./request');
 import path = require('path');
 import url = require('url');
+import log  = require('./log');
 var persistent = '.crpersistent';
 
 /**
@@ -84,6 +85,7 @@ function page(config: IConfig, address: string, done: (err: Error, result?: ISer
     var $ = cheerio.load(result);
     var title = $('span[itemprop=name]').text();
     if (!title) return done(new Error('Invalid page.(' + address + ')'));
+    log.info("Checking availability for " + title);
     var episodes: ISeriesEpisode[] = [];
     $('.episode').each((i, el) => {
       if ($(el).children('img[src*=coming_soon]').length) return;

src/video/stream.ts

@@ -2,6 +2,7 @@
 import childProcess = require('child_process');
 import path = require('path');
 import os = require('os');
+import log  = require('../log');
 
 /**
  * Streams the video to disk.
@@ -19,7 +20,7 @@ import os = require('os');
  	}
  	else if (mode == "HLS")
  	{
- 		console.info("Experimental FFMPEG, MAY FAIL!!!");
+ 		//log.debug("Experimental FFMPEG, MAY FAIL!!!");
  		var cmd=command("ffmpeg") + ' ' + 
  			'-i "' + rtmpInputPath + '" ' + 
  			'-c copy -bsf:a aac_adtstoasc ' + 
@@ -30,7 +31,7 @@ import os = require('os');
  	}
  	else
  	{
- 		console.error("No such mode: " + mode);
+ 		log.error("No such mode: " + mode);
  	}
 }
 

tsconfig.json

@@ -18,6 +18,7 @@
         "src/cli.ts",
         "src/episode.ts",
         "src/index.ts",
+        "src/log.ts",
         "src/interface/IConfig.d.ts",
         "src/interface/IConfigLine.d.ts",
         "src/interface/IConfigTask.d.ts",

tsd.json

@@ -1,33 +0,0 @@
-{
-  "version": "v4",
-  "repo": "borisyankov/DefinitelyTyped",
-  "ref": "master",
-  "path": "typings",
-  "bundle": "typings/tsd.d.ts",
-  "installed": {
-    "node/node.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "commander/commander.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "xml2js/xml2js.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "cheerio/cheerio.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "mkdirp/mkdirp.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "request/request.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "big-integer/big-integer.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    },
-    "form-data/form-data.d.ts": {
-      "commit": "3882d337bb0808cde9fe4c08012508a48c135482"
-    }
-  }
-}

tslint.json

@@ -47,8 +47,6 @@
     "no-var-requires": true,
     "one-line": [true,
       "check-catch",
-      "check-else",
-      "check-open-brace",
       "check-whitespace"
     ],
     "quotemark": [true, "single"],

typings.json

@@ -0,0 +1,13 @@
+{
+  "name": "crunchy",
+  "globalDependencies": {
+    "node": "github:DefinitelyTyped/DefinitelyTyped/node/node.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "commander": "github:DefinitelyTyped/DefinitelyTyped/commander/commander.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "xml2js": "github:DefinitelyTyped/DefinitelyTyped/xml2js/xml2js.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "cheerio": "github:DefinitelyTyped/DefinitelyTyped/cheerio/cheerio.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "mkdirp": "github:DefinitelyTyped/DefinitelyTyped/mkdirp/mkdirp.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "request": "github:DefinitelyTyped/DefinitelyTyped/request/request.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "big-integer": "github:DefinitelyTyped/DefinitelyTyped/big-integer/big-integer.d.ts#3882d337bb0808cde9fe4c08012508a48c135482",
+    "form-data": "github:DefinitelyTyped/DefinitelyTyped/form-data/form-data.d.ts#3882d337bb0808cde9fe4c08012508a48c135482"
+  }
+}