1.3.7

(Commit #200!)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -14,10 +14,12 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **Please fill theses informations:**
-(Add a X between brackets to make them ticked)
+(Add a X between brackets to make them ticked if relevant)
  - OS: [e.g:. Windows 10, Mac OS X 10.13, ...]
  - [ ] I'm using the latest version of Crunchy
  - [ ] I have a premium accrount on CR
+ - [ ] I am using a VPN
+ - My region in the world (country or continent):
  - Serie you get a problem with (and specify which episode if it is specific to one):
  - The command line you are running Crunchy with:
  - The message Crunchy is giving you, if any:
@@ -29,4 +31,5 @@ If applicable, add screenshots to help explain your problem.
 Add any other context about the problem here.
 
 
-_Also don't hesitate to add labels you feel apropriate on your report._
+_Also don't hesitate to add labels you feel apropriate on your report._
+_Please don't edit logs if you are adding them, apart from removing sensitive informations like login/password_

.gitignore

@@ -1,3 +1,2 @@
 dist/
 node_modules/
-typings/

README.md

@@ -66,62 +66,73 @@ The [command-line interface](http://en.wikipedia.org/wiki/Command-line_interface
 
     Options:
 
-       -V, --version         output the version number
-       -p, --pass <s>        The password.
-       -u, --user <s>        The e-mail address or username.
-       -c, --cache           Disables the cache.
-       -m, --merge           Disables merging subtitles and videos.
-       -e, --episodes <s>    Episode list. Read documentation on how to use
-       -f, --format <s>      The subtitle format. (Default: ass)
-       -o, --output <s>      The output path.
-       -s, --series <s>      The series override.
-       -n, --filename <s>    The name override.
-       -t, --tag <s>         The subgroup. (Default: CrunchyRoll) (default: CrunchyRoll)
-       -r, --resolution <s>  The video resolution. (Default: 1080 (360, 480, 720, 1080)) (default: 1080)
-       -g, --rebuildcrp      Rebuild the crpersistant file.
-       -b, --batch <s>       Batch file (default: CrunchyRoll.txt)
-       --verbose             Make tool verbose
-       --retry <i>           Number or time to retry fetching an episode. Default: 5 (default: 5)
-       -h, --help            output usage information
+        -V, --version         output the version number
+        -p, --pass <s>        The password.
+        -u, --user <s>        The e-mail address or username.
+        -c, --cache           Disables the cache.
+        -m, --merge           Disables merging subtitles and videos.
+        -e, --episodes <s>    Episode list. Read documentation on how to use
+        -f, --format <s>      The subtitle format. (default: ass)
+        -o, --output <s>      The output path.
+        -s, --series <s>      The series name override.
+        -n, --nametmpl <s>    Output name template (default: {SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - [{TAG}])
+        -t, --tag <s>         The subgroup. (default: CrunchyRoll)
+        -r, --resolution <s>  The video resolution. (valid: 360, 480, 720, 1080) (default: 1080)
+        -b, --batch <s>       Batch file (default: CrunchyRoll.txt)
+        --verbose             Make tool verbose
+        --rebuildcrp          Rebuild the crpersistant file.
+        --retry <i>           Number or time to retry fetching an episode. (default: 5)
+        -h, --help            output usage information
 
 #### Batch-mode
 
-When no sequence of series addresses is provided, the batch-mode source file will be read (which is *CrunchyRoll.txt* in the current work directory. Each line in this file is processed contain the URL of a series and can support some of the command line paramter (like *-e*). This makes it ideal to manage a large sequence of series addresses.
+When no sequence of series addresses is provided, the batch-mode source file will be read (which is *CrunchyRoll.txt* in the current work directory. Each line in this file is processed contain the URL of a series and can support some of the command line parameter (like `-e`). This makes it ideal to manage a large sequence of series addresses.
 
 #### Examples
 
 Download in batch-mode:
 
-    crunchy
+ You will need to create the batch file (default name is `CrunchyRoll.txt`):
 
-Download *Fairy Tail* to the current work directory:
+    http://www.cr.com/tail-fairy
+    http://www.cr.com/gin-mama
+    http://www.cr.com/two-parts
+    // Just download episodes 3 to 42
+    http://www.cr.com/defense-of-dwarfs -e 3-42
 
-    crunchy http://www.crunchyroll.com/fairy-tail
+ Then launch crunchy:
 
-Download *Fairy Tail* to `C:\Anime`:
+    crunchy -u login -p password http://www.cr.com/tail-fairy
 
-    crunchy --output C:\Anime http://www.crunchyroll.com/fairy-tail
+Download *Tail Fairy* to the current work directory:
 
-Download episode 42 of *Fairy Tail* to `C:\Anime`:
+    crunchy -u login -p password http://www.cr.com/tail-fairy
 
-    crunchy --output C:\Anime @http://www.crunchyroll.com/fairy-tail/episode-46-the-silver-labyrinth-662721
+Download *Tail Fairy* to `C:\Anime`:
+
+    crunchy -u login -p password --output C:\Anime http://www.cr.com/tail-fairy
+
+Download episode 42 of *Tail Fairy* to `C:\Anime`:
+
+    crunchy -u login -p password --output C:\Anime @http://www.cr.com/tail-fairy/episode-42-the-episode-which-dont-exist-665544
 
   *Notice the '@' in front of the URL, it is there to tell Crunchy that the URL is an episode URL and not a series URL.*
 
  or 
-    crunchy --output C:\Anime http://www.crunchyroll.com/fairy-tail -e 42
 
-Download episode 10 to 42 (both included) of *Fairy Tail*:
+    crunchy -u login -p password --output C:\Anime http://www.cr.com/tail-fairy -e 42
 
-    crunchy http://www.crunchyroll.com/fairy-tail -e 10-42
+Download episode 10 to 42 (both included) of *Tail Fairy*:
 
-Download episode up to 42 (included) of *Fairy Tail*:
+    crunchy -u login -p password http://www.cr.com/tail-fairy -e 10-42
 
-    crunchy http://www.crunchyroll.com/fairy-tail -e -42
+Download episode up to 42 (included) of *Tail Fairy*:
 
-Download episodes starting from 42 to the last available of *Fairy Tail*:
+    crunchy -u login -p password http://www.cr.com/tail-fairy -e -42
 
-    crunchy http://www.crunchyroll.com/fairy-tail -e 42-
+Download episodes starting from 42 to the last available of *Tail Fairy*:
+
+    crunchy -u login -p password http://www.cr.com/tail-fairy -e 42-
 
 
 

package.json

@@ -15,19 +15,18 @@
   "engines": {
     "node": ">=5.0"
   },
-  "version": "1.3.5",
+  "version": "1.3.7",
   "bin": {
     "crunchy": "./bin/crunchy",
     "crunchy.sh": "./bin/crunchy.sh"
   },
   "dependencies": {
-    "@types/node": "^10.3.3",
-    "big-integer": "^1.6.31",
+    "big-integer": "^1.6.32",
     "bluebird": "^3.5.1",
     "cheerio": "^0.22.0",
     "cloudscraper": "^1.5.0",
-    "commander": "^2.15.1",
-    "fs-extra": "^6.0.1",
+    "commander": "^2.16.0",
+    "fs-extra": "^7.0.0",
     "mkdirp": "^0.5.0",
     "pjson": "^1.0.9",
     "request": "^2.87.0",
@@ -35,15 +34,17 @@
     "xml2js": "^0.4.5"
   },
   "devDependencies": {
-    "@types/bluebird": "^3.5.20",
-    "@types/cheerio": "^0.22.7",
-    "@types/fs-extra": "^5.0.3",
+    "@types/bluebird": "^3.5.23",
+    "@types/cheerio": "^0.22.8",
+    "@types/fs-extra": "^5.0.4",
     "@types/mkdirp": "^0.5.2",
+    "@types/node": "^10.5.3",
     "@types/request": "^2.47.1",
-    "@types/request-promise": "^4.1.41",
+    "@types/request-promise": "^4.1.42",
     "@types/xml2js": "^0.4.3",
+    "npm-check": "^5.7.1",
     "tsconfig-lint": "^0.12.0",
-    "tslint": "^5.10.0",
+    "tslint": "^5.11.0",
     "typescript": "^2.9.2"
   },
   "scripts": {

src/batch.ts

@@ -62,6 +62,11 @@ export default function(args: string[], done: (err?: Error) => void)
         return done(err);
     }
 
+    if (tasksArr[0].address === '')
+    {
+        return done();
+    }
+
     let i = 0;
 
     (function next()
@@ -80,12 +85,18 @@ export default function(args: string[], done: (err?: Error) => void)
       {
         if (errin)
         {
+          if (errin.error)
+          {
+            /* Error from the request, so ignore it */
+            tasksArr[i].retry = 0;
+          }
+
           if (tasksArr[i].retry <= 0)
           {
-            log.error(errin.stack || errin);
+            log.error(JSON.stringify(errin));
             if (config.debug)
             {
-              log.dumpToDebug('BatchGiveUp', errin.stack || errin);
+              log.dumpToDebug('BatchGiveUp', JSON.stringify(errin));
             }
             log.error('Cannot get episodes from "' + tasksArr[i].address + '", please rerun later');
             /* Go to the next on the list */
@@ -95,11 +106,11 @@ export default function(args: string[], done: (err?: Error) => void)
           {
             if (config.verbose)
             {
-              log.error(errin);
+              log.error(JSON.stringify(errin));
             }
             if (config.debug)
             {
-              log.dumpToDebug('BatchRetry', errin.stack || errin);
+              log.dumpToDebug('BatchRetry', JSON.stringify(errin));
             }
             log.warn('Retrying to fetch episodes list from' + tasksArr[i].retry + ' / ' + config.retry);
             tasksArr[i].retry -= 1;
@@ -195,6 +206,27 @@ function get_max_filter(filter: string): number
   return +Infinity;
 }
 
+/**
+ * Check that URL start with http:// or https://
+ * As for some reason request just return an error but a useless one when that happen so check it
+ * soon enough.
+ */
+function checkURL(address: string): boolean
+{
+  if (address.startsWith('http:\/\/'))
+  {
+    return true;
+  }
+  if (address.startsWith('http:\/\/'))
+  {
+    return true;
+  }
+
+  log.error('URL ' + address + ' miss \'http:\/\/\' or \'https:\/\/\' => will be ignored');
+
+  return false;
+}
+
 /**
  * Parses the configuration or reads the batch-mode file for tasks.
  */
@@ -204,8 +236,13 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
   {
     return done(null, config.args.map((addressIn) =>
     {
-      return {address: addressIn, retry: config.retry,
-              episode_min: get_min_filter(config.episodes), episode_max: get_max_filter(config.episodes)};
+      if (checkURL(addressIn))
+      {
+        return {address: addressIn, retry: config.retry,
+                episode_min: get_min_filter(config.episodes), episode_max: get_max_filter(config.episodes)};
+      }
+
+      return {address: '', retry: 0, episode_min: 0, episode_max: 0};
     }));
   }
 
@@ -241,8 +278,11 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
             return;
           }
 
-          map.push({address: addressIn, retry: lineConfig.retry,
-                    episode_min: get_min_filter(lineConfig.episodes), episode_max: get_max_filter(lineConfig.episodes)});
+          if (checkURL(addressIn))
+          {
+            map.push({address: addressIn, retry: lineConfig.retry,
+                      episode_min: get_min_filter(lineConfig.episodes), episode_max: get_max_filter(lineConfig.episodes)});
+          }
         });
       });
       done(null, map);
@@ -265,17 +305,16 @@ function parse(args: string[]): IConfigLine
     // Episode filter
     .option('-e, --episodes <s>', 'Episode list. Read documentation on how to use')
     // Settings
-    .option('-f, --format <s>', 'The subtitle format. (Default: ass)')
+    .option('-f, --format <s>', 'The subtitle format.', 'ass')
     .option('-o, --output <s>', 'The output path.')
-    .option('-s, --series <s>', 'The series override.')
-    .option('-n, --filename <s>', 'The name override.')
-    .option('-t, --tag <s>', 'The subgroup. (Default: CrunchyRoll)', 'CrunchyRoll')
-    .option('-r, --resolution <s>', 'The video resolution. (Default: 1080 (360, 480, 720, 1080))',
-            '1080')
-    .option('-g, --rebuildcrp', 'Rebuild the crpersistant file.')
+    .option('-s, --series <s>', 'The series name override.')
+    .option('-n, --nametmpl <s>', 'Output name template', '{SERIES_TITLE} - s{SEASON_NUMBER}e{EPISODE_NUMBER} - {EPISODE_TITLE} - [{TAG}]')
+    .option('-t, --tag <s>', 'The subgroup.', 'CrunchyRoll')
+    .option('-r, --resolution <s>', 'The video resolution. (valid: 360, 480, 720, 1080)', '1080')
     .option('-b, --batch <s>', 'Batch file', 'CrunchyRoll.txt')
     .option('--verbose', 'Make tool verbose')
     .option('--debug', 'Create a debug file. Use only if requested!')
-    .option('--retry <i>', 'Number or time to retry fetching an episode. Default: 5', 5)
+    .option('--rebuildcrp', 'Rebuild the crpersistant file.')
+    .option('--retry <i>', 'Number or time to retry fetching an episode.', 5)
     .parse(args);
 }

src/episode.ts

@@ -74,11 +74,10 @@ function sanitiseFileName(str: string)
  */
 function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void)
 {
-  let series = config.series || page.series;
+  const serieFolder = sanitiseFileName(config.series || page.series);
 
-  series = sanitiseFileName(series);
-  let fileName = sanitiseFileName(name(config, page, series, ''));
-  let filePath = path.join(config.output || process.cwd(), series, fileName);
+  let fileName = sanitiseFileName(generateName(config, page));
+  let filePath = path.join(config.output || process.cwd(), serieFolder, fileName);
 
   if (fileExist(filePath + '.mkv'))
   {
@@ -95,8 +94,8 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
     do
     {
       count = count + 1;
-      fileName = sanitiseFileName(name(config, page, series, '-' + count));
-      filePath = path.join(config.output || process.cwd(), series, fileName);
+      fileName = sanitiseFileName(generateName(config, page, '-' + count));
+      filePath = path.join(config.output || process.cwd(), serieFolder, fileName);
     } while (fileExist(filePath + '.mkv'));
 
     log.warn('Renaming to \'' + fileName + '\'...');
@@ -114,6 +113,7 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
   {
     if (errM)
     {
+      log.dispEpisode(fileName, 'Error...', true);
       return done(errM, false);
     }
 
@@ -122,6 +122,7 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
     {
       if (errDS)
       {
+        log.dispEpisode(fileName, 'Error...', true);
         return done(errDS, false);
       }
 
@@ -133,6 +134,7 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
         {
           if (errDV)
           {
+            log.dispEpisode(fileName, 'Error...', true);
             return done(errDV, false);
           }
 
@@ -148,6 +150,7 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
           {
             if (errVM)
             {
+              log.dispEpisode(fileName, 'Error...', true);
               return done(errVM, false);
             }
 
@@ -211,19 +214,16 @@ function downloadVideo(config: IConfig,  page: IEpisodePage, player: IEpisodePla
 /**
  * Names the file based on the config, page, series and tag.
  */
-function name(config: IConfig, page: IEpisodePage, series: string, extra: string)
+function generateName(config: IConfig, page: IEpisodePage, extra = '')
 {
   const episodeNum = parseInt(page.episode, 10);
   const volumeNum = parseInt(page.volume, 10);
   const episode = (episodeNum < 10 ? '0' : '') + page.episode;
   const volume = (volumeNum < 10 ? '0' : '') + page.volume;
   const tag = config.tag || 'CrunchyRoll';
+  const series = config.series || page.series;
 
-  if (!page.filename) {
-    return page.series + ' - s' + volume + 'e' + episode + ' - [' + tag + ']' + extra;
-  }
-
-  return page.filename
+  return config.nametmpl
       .replace(/{EPISODE_ID}/g, page.id.toString())
       .replace(/{EPISODE_NUMBER}/g, episode)
       .replace(/{SEASON_NUMBER}/g, volume)

src/interface/IConfig.d.ts

@@ -10,7 +10,7 @@ interface IConfig {
   format?: string;
   output?: string;
   series?: string;
-  filename?: string;
+  nametmpl?: string;
   tag?: string;
   resolution?: string;
   video_format?: string;

src/log.ts

@@ -41,15 +41,9 @@ export function dumpToDebug(what: string, data: any, create = false)
 {
   if (create)
   {
-    fs.writeFile('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n', (err) =>
-    {
-      if (err) throw err;
-    });
+    fs.writeFileSync('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n');
     return;
   }
 
-  fs.appendFile('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n', (err) =>
-  {
-    if (err) throw err;
-  });
-}
+  fs.appendFileSync('debug.txt', '>>>>>>>> ' + what + ':\n' + data + '\n<<<<<<<<\n');
+}

src/my_request.ts

@@ -82,7 +82,7 @@ function login(sessionId: string, user: string, pass: string): Promise<any>
 /**
  * Performs a GET request for the resource.
  */
-export function get(config: IConfig, options: string|request.Options, done: (err: Error, result?: string) => void)
+export function get(config: IConfig, options: string|request.Options, done: (err: any, result?: string) => void)
 {
   authenticate(config, (err) =>
   {
@@ -91,7 +91,7 @@ export function get(config: IConfig, options: string|request.Options, done: (err
       return done(err);
     }
 
-    cloudscraper.request(modify(options, 'GET'), (error: Error, response: any, body: any) =>
+    cloudscraper.request(modify(options, 'GET'), (error: any, response: any, body: any) =>
     {
       if (error) return done(error);
       done(null, typeof body === 'string' ? body : String(body));
@@ -223,4 +223,4 @@ function modify(options: string|request.Options, reqMethod: string): request.Opt
     url: options.toString(),
     method: reqMethod
   };
-}
+}

src/series.ts

@@ -27,7 +27,7 @@ function fileExist(path: string)
 /**
  * Streams the series to disk.
  */
-export default function(config: IConfig, task: IConfigTask, done: (err: Error) => void)
+export default function(config: IConfig, task: IConfigTask, done: (err: any) => void)
 {
   const persistentPath = path.join(config.output || process.cwd(), persistent);
 
@@ -45,6 +45,12 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
     {
       if (errP)
       {
+        const reqErr = errP.error;
+        if ((reqErr.syscall === 'getaddrinfo') && (reqErr.errno === 'ENOTFOUND'))
+        {
+          log.error('The URL \'' + task.address + '\' is invalid, please check => I\'m ignoring it.');
+        }
+
         return done(errP);
       }
 
@@ -61,10 +67,17 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
         {
           if (errD)
           {
+            /* Check if domain is valid */
+            const reqErr = errD.error;
+            if ((reqErr.syscall === 'getaddrinfo') && (reqErr.errno === 'ENOTFOUND'))
+            {
+               page.episodes[i].retry = 0;
+               log.error('The URL \'' + task.address + '\' is invalid, please check => I\'m ignoring it.');
+            }
+
             if (page.episodes[i].retry <= 0)
             {
-              log.dispEpisode(config.filename, 'Error...', true);
-              log.error(errD);
+              log.error(JSON.stringify(errD));
               log.error('Cannot fetch episode "s' + page.episodes[i].volume + 'e' + page.episodes[i].episode +
                             '", please rerun later');
               /* Go to the next on the list */
@@ -72,13 +85,12 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
             }
             else
             {
-              log.dispEpisode(config.filename, 'Error...', true);
               if ((config.verbose) || (config.debug))
               {
                 if (config.debug)
                 {
                   log.dumpToDebug('series address', task.address);
-                  log.dumpToDebug('series error', errD.stack || errD);
+                  log.dumpToDebug('series error', JSON.stringify(errD));
                   log.dumpToDebug('series data', JSON.stringify(page));
                 }
                 log.error(errD);
@@ -122,7 +134,7 @@ export default function(config: IConfig, task: IConfigTask, done: (err: Error) =
  */
 function download(cache: {[address: string]: number}, config: IConfig,
                   task: IConfigTask, item: ISeriesEpisode,
-                  done: (err: Error, ign: boolean) => void)
+                  done: (err: any, ign: boolean) => void)
 {
   const episodeNumber = parseInt(item.episode, 10);
 
@@ -154,7 +166,7 @@ function download(cache: {[address: string]: number}, config: IConfig,
 /**
  * Requests the page and scrapes the episodes and series.
  */
-function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, result?: ISeries) => void)
+function pageScrape(config: IConfig, task: IConfigTask, done: (err: any, result?: ISeries) => void)
 {
   if (task.address[0] === '@')
   {
@@ -172,7 +184,8 @@ function pageScrape(config: IConfig, task: IConfigTask, done: (err: Error, resul
   {
     let episodeCount = 0;
     my_request.get(config, task.address, (err, result) => {
-      if (err) {
+      if (err)
+      {
         return done(err);
       }