diff --git a/LICENSE b/LICENSE index 106aee2..f4606b3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ Copyright (c) 2015 Roel van Uden +Copyright (c) 2016 Manoel Trapier Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to diff --git a/README b/README new file mode 120000 index 0000000..42061c0 --- /dev/null +++ b/README @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/README.md b/README.md index 3fa719c..5992020 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# CrunchyRoll.js +# Crunchy: a fork of Deathspike/CrunchyRoll.js -*CrunchyRoll.js* is capable of downloading *anime* episodes from the popular *CrunchyRoll* streaming service. An episode is stored in the original video format (often H.264 in a MP4 container) and the configured subtitle format (ASS or SRT).The two output files are then merged into a single MKV file. +*Crunchy* is capable of downloading *anime* episodes from the popular *CrunchyRoll* streaming service. An episode is stored in the original video format (often H.264 in a MP4 container) and the configured subtitle format (ASS or SRT).The two output files are then merged into a single MKV file. ## Motivation @@ -10,6 +10,8 @@ This application is not endorsed or affliated with *CrunchyRoll*. The usage of this application enables episodes to be downloaded for offline convenience which may be forbidden by law in your country. Usage of this application may also cause a violation of the agreed *Terms of Service* between you and the stream provider. A tool is not responsible for your actions; please make an informed decision prior to using this application. +**PLEASE _ONLY_ USE THIS TOOL IF YOU HAVE A _PREMIUM ACCOUNT_** + ## Configuration It is recommended to enable authentication (`-p` and `-u`) so your account permissions and settings are available for use. It is not possible to download non-free material without an account and premium subscription. Furthermore, the default account settings are used when downloading. If you want the highest quality videos, configure these preferences at https://www.crunchyroll.com/acct/?action=video. @@ -26,30 +28,30 @@ Use the applicable instructions to install. Is your operating system not listed? ### Debian (Mint, Ubuntu, etc) -1. Run in *Terminal*: `sudo apt-get install nodejs npm mkvtoolnix rtmpdump` +1. Run in *Terminal*: `sudo apt-get install nodejs npm mkvtoolnix rtmpdump ffmpeg` 2. Run in *Terminal*: `sudo ln -s /usr/bin/nodejs /usr/bin/node` -3. Run in *Terminal*: `sudo npm install -g crunchyroll` +3. Run in *Terminal*: `sudo npm install -g crunchy` ### Mac OS X 1. Install *Homebrew* following the instructions at http://brew.sh/ -2. Run in *Terminal*: `brew install node mkvtoolnix rtmpdump` -3. Run in *Terminal*: `npm install -g crunchyroll` +2. Run in *Terminal*: `brew install node mkvtoolnix rtmpdump ffmpeg` +3. Run in *Terminal*: `npm install -g crunchy` ### Windows 1. Install *NodeJS* following the instructions at http://nodejs.org/ -3. Run in *Command Prompt*: `npm install -g crunchyroll` +3. Run in *Command Prompt*: `npm install -g crunchy` ## Instructions Use the applicable instructions for the interface of your choice (currently limited to command-line). -### Command-line Interface (`crunchyroll`) +### Command-line Interface (`crunchy`) -The [command-line interface](http://en.wikipedia.org/wiki/Command-line_interface) does not have a graphical component and is ideal for automation purposes and headless machines. The interface can run using a sequence of series addresses (the site address containing the episode listing), or with a batch-mode source file. The `crunchyroll --help` command will produce the following output: +The [command-line interface](http://en.wikipedia.org/wiki/Command-line_interface) does not have a graphical component and is ideal for automation purposes and headless machines. The interface can run using a sequence of series addresses (the site address containing the episode listing), or with a batch-mode source file. The `crunchy --help` command will produce the following output: - Usage: crunchyroll [options] + Usage: crunchy [options] Options: @@ -74,15 +76,15 @@ When no sequence of series addresses is provided, the batch-mode source file wil Download in batch-mode: - crunchyroll + crunchy Download *Fairy Tail* to the current work directory: - crunchyroll http://www.crunchyroll.com/fairy-tail + crunchy http://www.crunchyroll.com/fairy-tail Download *Fairy Tail* to `C:\Anime`: - crunchyroll --output C:\Anime http://www.crunchyroll.com/fairy-tail + crunchy --output C:\Anime http://www.crunchyroll.com/fairy-tail #### Switches diff --git a/bin/crunchyroll b/bin/crunchy similarity index 100% rename from bin/crunchyroll rename to bin/crunchy diff --git a/bin/ffmpeg.exe b/bin/ffmpeg.exe new file mode 100755 index 0000000..9b1df89 Binary files /dev/null and b/bin/ffmpeg.exe differ diff --git a/package.json b/package.json index 96c0a95..b5735ea 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,20 @@ { - "author": "Roel van Uden", - "description": "CrunchyRoll.js is capable of downloading anime episodes from the popular CrunchyRoll streaming service.", + "author": "Godzil", + "description": "Crunchy.js is a fork of Crunchyroll.js, capable of downloading anime episodes from the popular CrunchyRoll streaming service.", + "license": "MIT", "keywords": [ "anime", "download", "crunchyroll" ], - "name": "crunchyroll", + "name": "crunchy", "repository": { "type": "git", - "url": "git://github.com/Deathspike/crunchyroll.js.git" + "url": "git://github.com/Godzil/crunchyroll.js.git" }, - "version": "1.1.5", + "version": "1.1.11", "bin": { - "crunchyroll": "./bin/crunchyroll" + "crunchy": "./bin/crunchy" }, "dependencies": { "big-integer": "1.4.4", @@ -30,7 +31,7 @@ }, "scripts": { "prepublish": "npm run tsd && tsc", - "test": "node ts --only-test", + "test": "node ts --only-test", "tsd": "tsd reinstall -o -s" } } diff --git a/src/episode.ts b/src/episode.ts index a2eeacb..9a86472 100644 --- a/src/episode.ts +++ b/src/episode.ts @@ -11,11 +11,11 @@ import xml2js = require('xml2js'); /** * Streams the episode to disk. */ -export default function(config: IConfig, address: string, done: (err: Error) => void) { +export default function(config: IConfig, address: string, done: (err: Error, ign: boolean) => void) { scrapePage(config, address, (err, page) => { - if (err) return done(err); + if (err) return done(err, false); scrapePlayer(config, address, page.id, (err, player) => { - if (err) return done(err); + if (err) return done(err, false); download(config, page, player, done); }); }); @@ -24,37 +24,72 @@ export default function(config: IConfig, address: string, done: (err: Error) => /** * Completes a download and writes the message with an elapsed time. */ -function complete(message: string, begin: number, done: (err: Error) => void) { +function complete(message: string, begin: number, done: (err: Error, ign: boolean) => void) { var timeInMs = Date.now() - begin; var seconds = prefix(Math.floor(timeInMs / 1000) % 60, 2); var minutes = prefix(Math.floor(timeInMs / 1000 / 60) % 60, 2); var hours = prefix(Math.floor(timeInMs / 1000 / 60 / 60), 2); console.log(message + ' (' + hours + ':' + minutes + ':' + seconds + ')'); - done(null); + done(null, false); +} + +/** + * Check if a file exist.. + */ +function fileExist(path: string) { + try + { + fs.statSync(path); + return true; + } + catch (e) { } + return false; } /** * Downloads the subtitle and video. */ -function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error) => void) { +function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void) { var series = config.series || page.series; - var fileName = name(config, page, series); + series = series.replace("/","_").replace("'","_").replace(":","_"); + var fileName = name(config, page, series, "").replace("/","_").replace("'","_").replace(":","_"); var filePath = path.join(config.output || process.cwd(), series, fileName); + if (fileExist(filePath + ".mkv")) + { + var count = 0; + console.info("File '"+fileName+"' already exist..."); + do + { + count = count + 1; + fileName = name(config, page, series, "-" + count).replace("/","_").replace("'","_").replace(":","_"); + filePath = path.join(config.output || process.cwd(), series, fileName); + } while(fileExist(filePath + ".mkv")) + console.info("Renaming to '"+fileName+"'..."); + } + mkdirp(path.dirname(filePath), (err: Error) => { - if (err) return done(err); + if (err) return done(err, false); downloadSubtitle(config, player, filePath, err => { - if (err) return done(err); + if (err) return done(err, false); var now = Date.now(); - console.log('Fetching ' + fileName); - downloadVideo(config, page, player, filePath, err => { - if (err) return done(err); - if (config.merge) return complete('Finished ' + fileName, now, done); - var isSubtited = Boolean(player.subtitle); - video.merge(config, isSubtited, player.video.file, filePath, err => { - if (err) return done(err); - complete('Finished ' + fileName, now, done); + if (player.video.file != undefined) + { + console.log('Fetching ' + fileName); + downloadVideo(config, page, player, filePath, err => { + if (err) return done(err, false); + if (config.merge) return complete('Finished ' + fileName, now, done); + var isSubtited = Boolean(player.subtitle); + video.merge(config, isSubtited, player.video.file, filePath, player.video.mode, err => { + if (err) return done(err, false); + complete('Finished ' + fileName, now, done); + }); }); - }); + } + else + { + console.log('Ignoring ' + fileName + ': not released yet'); + done(null, true); + } }); }); } @@ -88,18 +123,21 @@ function downloadVideo(config: IConfig, player.video.host, player.video.file, page.swf, - filePath + path.extname(player.video.file), + filePath, path.extname(player.video.file), + player.video.mode, done); } /** * Names the file based on the config, page, series and tag. */ -function name(config: IConfig, page: IEpisodePage, series: string) { - var episode = (page.episode < 10 ? '0' : '') + page.episode; - var volume = (page.volume < 10 ? '0' : '') + page.volume; +function name(config: IConfig, page: IEpisodePage, series: string, extra: string) { + var episodeNum = parseInt(page.episode, 10); + var volumeNum = parseInt(page.volume, 10); + var episode = (episodeNum < 10 ? '0' : '') + page.episode; + var volume = (volumeNum < 10 ? '0' : '') + page.volume; var tag = config.tag || 'CrunchyRoll'; - return series + ' ' + volume + 'x' + episode + ' [' + tag + ']'; + return series + ' - s' + volume + 'e' + episode +' - [' + tag + ']' + extra; } /** @@ -121,15 +159,29 @@ function scrapePage(config: IConfig, address: string, done: (err: Error, page?: if (err) return done(err); var $ = cheerio.load(result); var swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href')); - var regexp = /-\s+(?:Watch\s+)?(.+?)(?:\s+Season\s+([0-9]+))?(?:\s+-)?\s+Episode\s+([0-9]+)/; - var data = regexp.exec($('title').text()); - if (!swf || !data) return done(new Error('Invalid page.')); + var regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/; + var look = $('#showmedia_about_media').text(); + var seasonTitle = $('span[itemprop="title"]').text(); + var data = regexp.exec(look); + + if (!swf || !data) + { + console.info('Something wrong in the page at '+address+' (data are: '+look+')'); + console.info('Setting Season to 0 and episode to \’0\’...'); + done(null, { + id: id, + episode: "0", + series: seasonTitle, + swf: swf[1], + volume: "0" + }); + } done(null, { id: id, - episode: parseInt(data[3], 10), + episode: data[3], series: data[1], swf: swf[1], - volume: parseInt(data[2], 10) || 1 + volume: data[2] || "1" }); }); } @@ -152,6 +204,11 @@ function scrapePlayer(config: IConfig, address: string, id: number, done: (err: if (err) return done(err); try { var isSubtitled = Boolean(player['default:preload'].subtitle); + var streamMode="RTMP"; + if (player['default:preload'].stream_info.host == "") + { + streamMode="HLS"; + } done(null, { subtitle: isSubtitled ? { id: parseInt(player['default:preload'].subtitle.$.id, 10), @@ -159,6 +216,7 @@ function scrapePlayer(config: IConfig, address: string, id: number, done: (err: data: player['default:preload'].subtitle.data } : null, video: { + mode: streamMode, file: player['default:preload'].stream_info.file, host: player['default:preload'].stream_info.host } diff --git a/src/interface/IEpisodePage.d.ts b/src/interface/IEpisodePage.d.ts index f3618dc..6481cdb 100644 --- a/src/interface/IEpisodePage.d.ts +++ b/src/interface/IEpisodePage.d.ts @@ -1,7 +1,7 @@ interface IEpisodePage { id: number; - episode: number; + episode: string; series: string; - volume: number; + volume: string; swf: string; } diff --git a/src/interface/IEpisodePlayer.d.ts b/src/interface/IEpisodePlayer.d.ts index 67379ad..f004964 100644 --- a/src/interface/IEpisodePlayer.d.ts +++ b/src/interface/IEpisodePlayer.d.ts @@ -5,6 +5,7 @@ interface IEpisodePlayer { data: string; }; video: { + mode: string; file: string; host: string; }; diff --git a/src/interface/ISeriesEpisode.d.ts b/src/interface/ISeriesEpisode.d.ts index 521167e..bb38ac8 100644 --- a/src/interface/ISeriesEpisode.d.ts +++ b/src/interface/ISeriesEpisode.d.ts @@ -1,5 +1,5 @@ interface ISeriesEpisode { address: string; - episode: number; + episode: string; volume: number; } diff --git a/src/request.ts b/src/request.ts index 84d60c2..a601df4 100644 --- a/src/request.ts +++ b/src/request.ts @@ -97,4 +97,4 @@ function modify(options: string|request.Options): request.Options { return options; } return {jar: true, headers: defaultHeaders, url: options.toString()}; -} \ No newline at end of file +} diff --git a/src/series.ts b/src/series.ts index 65d2994..16fc584 100644 --- a/src/series.ts +++ b/src/series.ts @@ -19,14 +19,22 @@ export default function(config: IConfig, address: string, done: (err: Error) => var i = 0; (function next() { if (i >= page.episodes.length) return done(null); - download(cache, config, address, page.episodes[i], err => { + download(cache, config, address, page.episodes[i], (err, ignored) => { if (err) return done(err); - var newCache = JSON.stringify(cache, null, ' '); - fs.writeFile(persistentPath, newCache, err => { - if (err) return done(err); + if ((ignored == false) || (ignored == undefined)) + { + var newCache = JSON.stringify(cache, null, ' '); + fs.writeFile(persistentPath, newCache, err => { + if (err) return done(err); + i += 1; + next(); + }); + } + else + { i += 1; next(); - }); + } }); })(); }); @@ -40,14 +48,14 @@ function download(cache: {[address: string]: number}, config: IConfig, baseAddress: string, item: ISeriesEpisode, - done: (err: Error) => void) { - if (!filter(config, item)) return done(null); + done: (err: Error, ign: boolean) => void) { + if (!filter(config, item)) return done(null, false); var address = url.resolve(baseAddress, item.address); - if (cache[address]) return done(null); - episode(config, address, err => { - if (err) return done(err); + if (cache[address]) return done(null, false); + episode(config, address, (err, ignored) => { + if (err) return done(err, false); cache[address] = Date.now(); - done(null); + done(null, ignored); }); } @@ -57,8 +65,8 @@ function download(cache: {[address: string]: number}, function filter(config: IConfig, item: ISeriesEpisode) { // Filter on chapter. var episodeFilter = config.episode; - if (episodeFilter > 0 && item.episode <= episodeFilter) return false; - if (episodeFilter < 0 && item.episode >= -episodeFilter) return false; + if (episodeFilter > 0 && parseInt(item.episode, 10) <= episodeFilter) return false; + if (episodeFilter < 0 && parseInt(item.episode, 10) >= -episodeFilter) return false; // Filter on volume. var volumeFilter = config.volume; @@ -75,18 +83,18 @@ function page(config: IConfig, address: string, done: (err: Error, result?: ISer if (err) return done(err); var $ = cheerio.load(result); var title = $('span[itemprop=name]').text(); - if (!title) return done(new Error('Invalid page.')); + if (!title) return done(new Error('Invalid page.(' + address + ')')); var episodes: ISeriesEpisode[] = []; $('.episode').each((i, el) => { if ($(el).children('img[src*=coming_soon]').length) return; var volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text()); - var regexp = /Episode\s+([0-9]+)\s*$/i; + var regexp = /Episode\s+((PV )?[S0-9][P0-9.]*[a-fA-F]?)\s*$/i; var episode = regexp.exec($(el).children('.series-title').text()); var address = $(el).attr('href'); if (!address || !episode) return; episodes.push({ address: address, - episode: parseInt(episode[0], 10), + episode: episode[1], volume: volume ? parseInt(volume[0], 10) : 1 }); }); diff --git a/src/video/merge.ts b/src/video/merge.ts index 8cfc643..ea366d6 100644 --- a/src/video/merge.ts +++ b/src/video/merge.ts @@ -8,9 +8,17 @@ import subtitle from '../subtitle/index'; /** * Merges the subtitle and video files into a Matroska Multimedia Container. */ - export default function(config: IConfig, isSubtitled: boolean, rtmpInputPath: string, filePath: string, done: (err: Error) => void) { + export default function(config: IConfig, isSubtitled: boolean, rtmpInputPath: string, filePath: string, streamMode: string, done: (err: Error) => void) { var subtitlePath = filePath + '.' + (subtitle.formats[config.format] ? config.format : 'ass'); - var videoPath = filePath + path.extname(rtmpInputPath); + var videoPath = filePath; + if (streamMode == "RTMP") + { + videoPath += path.extname(rtmpInputPath); + } + else + { + videoPath += ".mp4"; + } childProcess.exec(command() + ' ' + '-o "' + filePath + '.mkv" ' + '"' + videoPath + '" ' + diff --git a/src/video/stream.ts b/src/video/stream.ts index 6a116c5..0bfc858 100644 --- a/src/video/stream.ts +++ b/src/video/stream.ts @@ -6,20 +6,38 @@ import os = require('os'); /** * Streams the video to disk. */ - export default function(rtmpUrl: string, rtmpInputPath: string, swfUrl: string, filePath: string, done: (err: Error) => void) { - childProcess.exec(command() + ' ' + - '-r "' + rtmpUrl + '" ' + - '-y "' + rtmpInputPath + '" ' + - '-W "' + swfUrl + '" ' + - '-o "' + filePath + '"', { - maxBuffer: Infinity - }, done); + export default function(rtmpUrl: string, rtmpInputPath: string, swfUrl: string, filePath: string, fileExt: string, mode: string, done: (err: Error) => void) { + if (mode == "RTMP") + { + childProcess.exec(command("rtmpdump") + ' ' + + '-r "' + rtmpUrl + '" ' + + '-y "' + rtmpInputPath + '" ' + + '-W "' + swfUrl + '" ' + + '-o "' + filePath + fileExt + '"', { + maxBuffer: Infinity + }, done); + } + else if (mode == "HLS") + { + console.info("Experimental FFMPEG, MAY FAIL!!!"); + var cmd=command("ffmpeg") + ' ' + + '-i "' + rtmpInputPath + '" ' + + '-c copy -bsf:a aac_adtstoasc ' + + '"' + filePath + '.mp4"'; + childProcess.exec(cmd, { + maxBuffer: Infinity + }, done); + } + else + { + console.error("No such mode: " + mode); + } } /** * Determines the command for the operating system. */ -function command(): string { - if (os.platform() !== 'win32') return 'rtmpdump'; - return '"' + path.join(__dirname, '../../bin/rtmpdump.exe') + '"'; +function command(exe: string): string { + if (os.platform() !== 'win32') return exe; + return '"' + path.join(__dirname, '../../bin/' + exe + '.exe') + '"'; }