godzil/crunchy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update npm packages, cleanup the code, cleanup all tslint complain

Browse Source
This commit is contained in:
Godzil 2017-02-10 17:43:52 +00:00
parent 5d9c25491d
commit bee3f33e20
13 changed files with 751 additions and 389 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
package.json
View File

@ -17,22 +17,29 @@
"crunchy": "./bin/crunchy"
},
"dependencies": {
"big-integer": "1.4.4",
"cheerio": "0.22.0",
"cloudscraper": "1.4.1",
"commander": "2.6.0",
"mkdirp": "0.5.0",
"request": "2.74.0",
"xml2js": "0.4.5"
"big-integer": "^1.4.4",
"cheerio": "^0.22.0",
"cloudscraper": "^1.4.1",
"commander": "^2.6.0",
"mkdirp": "^0.5.0",
"request": "^2.74.0",
"xml2js": "^0.4.5"
},
"devDependencies": {
"typings": "2.1.0",
"tslint": "2.3.0-beta",
"typescript": "1.5.0-beta"
"tsconfig-lint": "^0.12.0",
"tslint": "^4.4.2",
"typescript": "^2.2.0",
"typings": "^2.1.0"
},
"scripts": {
"prepublish": "npm run types && tsc",
"test": "node ts --only-test",
"types": "typings install"
"compile": "tsc",
"test": "tslint -c ./tslint.json --project ./tsconfig.json ./src/**/*.ts",
"types": "typings install",
"reinstall": "tsd reinstall; npm run types",
"start": "node ./bin/crunchy"
},
"bugs": {
"url": "https://github.com/Godzil/Crunchy/issues"
}
}

126
src/batch.ts
View File

@ -7,16 +7,33 @@ import series from './series';
/**
* Streams the batch of series to disk.
*/
export default function(args: string[], done: (err?: Error) => void) {
var config = parse(args);
var batchPath = path.join(config.output || process.cwd(), 'CrunchyRoll.txt');
tasks(config, batchPath, (err, tasks) => {
if (err) return done(err);
var i = 0;
(function next() {
if (i >= tasks.length) return done();
series(tasks[i].config, tasks[i].address, err => {
if (err) return done(err);
export default function(args: string[], done: (err?: Error) => void)
{
const config = parse(args);
const batchPath = path.join(config.output || process.cwd(), 'CrunchyRoll.txt');
tasks(config, batchPath, (err, tasks) =>
{
if (err)
{
return done(err);
}
let i = 0;
(function next()
{
if (i >= tasks.length)
{
return done();
}
series(tasks[i].config, tasks[i].address, err =>
{
if (err)
{
return done(err);
}
i += 1;
next();
});
@ -27,42 +44,82 @@ export default function(args: string[], done: (err?: Error) => void) {
/**
* Splits the value into arguments.
*/
function split(value: string): string[] {
var inQuote = false;
var i: number;
var pieces: string[] = [];
var previous = 0;
for (i = 0; i < value.length; i += 1) {
if (value.charAt(i) === '"') inQuote = !inQuote;
if (!inQuote && value.charAt(i) === ' ') {
function split(value: string): string[]
{
let inQuote = false;
let i: number;
let pieces: string[] = [];
let previous = 0;
for (i = 0; i < value.length; i += 1)
{
if (value.charAt(i) === '"')
{
inQuote = !inQuote;
}
if (!inQuote && value.charAt(i) === ' ')
{
pieces.push(value.substring(previous, i).match(/^"?(.+?)"?$/)[1]);
previous = i + 1;
}
}
var lastPiece = value.substring(previous, i).match(/^"?(.+?)"?$/);
if (lastPiece) pieces.push(lastPiece[1]);
let lastPiece = value.substring(previous, i).match(/^"?(.+?)"?$/);
if (lastPiece)
{
pieces.push(lastPiece[1]);
}
return pieces;
}
/**
* Parses the configuration or reads the batch-mode file for tasks.
*/
function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?: IConfigTask[]) => void) {
if (config.args.length) {
return done(null, config.args.map(address => {
function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?: IConfigTask[]) => void)
{
if (config.args.length)
{
return done(null, config.args.map(address =>
{
return {address: address, config: config};
}));
}
fs.exists(batchPath, exists => {
if (!exists) return done(null, []);
fs.readFile(batchPath, 'utf8', (err, data) => {
if (err) return done(err);
var map: IConfigTask[] = [];
data.split(/\r?\n/).forEach(line => {
if (/^(\/\/|#)/.test(line)) return;
var lineConfig = parse(process.argv.concat(split(line)));
lineConfig.args.forEach(address => {
if (!address) return;
fs.exists(batchPath, exists =>
{
if (!exists)
{
return done(null, []);
}
fs.readFile(batchPath, 'utf8', (err, data) =>
{
if (err)
{
return done(err);
}
let map: IConfigTask[] = [];
data.split(/\r?\n/).forEach(line =>
{
if (/^(\/\/|#)/.test(line))
{
return;
}
let lineConfig = parse(process.argv.concat(split(line)));
lineConfig.args.forEach(address =>
{
if (!address)
{
return;
}
map.push({address: address, config: lineConfig});
});
});
@ -74,7 +131,8 @@ function tasks(config: IConfigLine, batchPath: string, done: (err: Error, tasks?
/**
* Parses the arguments and returns a configuration.
*/
function parse(args: string[]): IConfigLine {
function parse(args: string[]): IConfigLine
{
return new commander.Command().version(require('../package').version)
// Authentication
.option('-p, --pass <s>', 'The password.')

8
src/cli.ts
View File

@ -1,6 +1,10 @@
'use strict';
import batch from './batch';
batch(process.argv, (err: any) => {
if (err) console.error(err.stack || err);
batch(process.argv, (err: any) =>
{
if (err)
{
console.error(err.stack || err);
}
});

295
src/episode.ts
View File

@ -12,11 +12,22 @@ import log = require('./log');
/**
* Streams the episode to disk.
*/
export default function(config: IConfig, address: string, done: (err: Error, ign: boolean) => void) {
scrapePage(config, address, (err, page) => {
if (err) return done(err, false);
scrapePlayer(config, address, page.id, (err, player) => {
if (err) return done(err, false);
export default function(config: IConfig, address: string, done: (err: Error, ign: boolean) => void)
{
scrapePage(config, address, (err, page) =>
{
if (err)
{
return done(err, false);
}
scrapePlayer(config, address, page.id, (err, player) =>
{
if (err)
{
return done(err, false);
}
download(config, page, player, done);
});
});
@ -25,63 +36,98 @@ export default function(config: IConfig, address: string, done: (err: Error, ign
/**
* Completes a download and writes the message with an elapsed time.
*/
function complete(epName: string, message: string, begin: number, done: (err: Error, ign: boolean) => void) {
var timeInMs = Date.now() - begin;
var seconds = prefix(Math.floor(timeInMs / 1000) % 60, 2);
var minutes = prefix(Math.floor(timeInMs / 1000 / 60) % 60, 2);
var hours = prefix(Math.floor(timeInMs / 1000 / 60 / 60), 2);
function complete(epName: string, message: string, begin: number, done: (err: Error, ign: boolean) => void)
{
const timeInMs = Date.now() - begin;
const seconds = prefix(Math.floor(timeInMs / 1000) % 60, 2);
const minutes = prefix(Math.floor(timeInMs / 1000 / 60) % 60, 2);
const hours = prefix(Math.floor(timeInMs / 1000 / 60 / 60), 2);
log.dispEpisode(epName, message + ' (' + hours + ':' + minutes + ':' + seconds + ')', true);
done(null, false);
}
/**
* Check if a file exist..
*/
function fileExist(path: string) {
function fileExist(path: string)
{
try
{
fs.statSync(path);
return true;
}
catch (e) { }
return false;
fs.statSync(path);
return true;
} catch (e)
{
return false;
}
}
/**
* Downloads the subtitle and video.
*/
function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void) {
var series = config.series || page.series;
series = series.replace("/","_").replace("'","_").replace(":","_");
var fileName = name(config, page, series, "").replace("/","_").replace("'","_").replace(":","_");
var filePath = path.join(config.output || process.cwd(), series, fileName);
if (fileExist(filePath + ".mkv"))
function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, done: (err: Error, ign: boolean) => void)
{
let series = config.series || page.series;
series = series.replace('/', '_').replace('\'', '_').replace(':', '_');
let fileName = name(config, page, series, '').replace('/', '_').replace('\'', '_').replace(':', '_');
let filePath = path.join(config.output || process.cwd(), series, fileName);
if (fileExist(filePath + '.mkv'))
{
var count = 0;
log.warn("File '"+fileName+"' already exist...");
let count = 0;
log.warn('File \'' + fileName + '\' already exist...');
do
{
count = count + 1;
fileName = name(config, page, series, "-" + count).replace("/","_").replace("'","_").replace(":","_");
fileName = name(config, page, series, '-' + count).replace('/', '_').replace('\'', '_').replace(':', '_');
filePath = path.join(config.output || process.cwd(), series, fileName);
} while(fileExist(filePath + ".mkv"))
log.warn("Renaming to '"+fileName+"'...");
} while (fileExist(filePath + '.mkv'));
log.warn('Renaming to \'' + fileName + '\'...');
}
mkdirp(path.dirname(filePath), (err: Error) => {
if (err) return done(err, false);
downloadSubtitle(config, player, filePath, err => {
if (err) return done(err, false);
var now = Date.now();
if (player.video.file != undefined)
{
mkdirp(path.dirname(filePath), (err: Error) =>
{
if (err)
{
return done(err, false);
}
downloadSubtitle(config, player, filePath, err =>
{
if (err)
{
return done(err, false);
}
const now = Date.now();
if (player.video.file !== undefined)
{
log.dispEpisode(fileName, 'Fetching...', false);
downloadVideo(config, page, player, filePath, err => {
if (err) return done(err, false);
if (config.merge) return complete(fileName, 'Finished!', now, done);
var isSubtited = Boolean(player.subtitle);
video.merge(config, isSubtited, player.video.file, filePath, player.video.mode, err => {
if (err) return done(err, false);
downloadVideo(config, page, player, filePath, err =>
{
if (err)
{
return done(err, false);
}
if (config.merge)
{
return complete(fileName, 'Finished!', now, done);
}
const isSubtited = Boolean(player.subtitle);
video.merge(config, isSubtited, player.video.file, filePath, player.video.mode, err =>
{
if (err)
{
return done(err, false);
}
complete(fileName, 'Finished!', now, done);
});
});
@ -98,15 +144,32 @@ function download(config: IConfig, page: IEpisodePage, player: IEpisodePlayer, d
/**
* Saves the subtitles to disk.
*/
function downloadSubtitle(config: IConfig, player: IEpisodePlayer, filePath: string, done: (err?: Error) => void) {
var enc = player.subtitle;
if (!enc) return done();
subtitle.decode(enc.id, enc.iv, enc.data, (err, data) => {
if (err) return done(err);
var formats = subtitle.formats;
var format = formats[config.format] ? config.format : 'ass';
formats[format](data, (err: Error, decodedSubtitle: string) => {
if (err) return done(err);
function downloadSubtitle(config: IConfig, player: IEpisodePlayer, filePath: string, done: (err?: Error) => void)
{
const enc = player.subtitle;
if (!enc)
{
return done();
}
subtitle.decode(enc.id, enc.iv, enc.data, (err, data) =>
{
if (err)
{
return done(err);
}
const formats = subtitle.formats;
const format = formats[config.format] ? config.format : 'ass';
formats[format](data, (err: Error, decodedSubtitle: string) =>
{
if (err)
{
return done(err);
}
fs.writeFile(filePath + '.' + format, '\ufeff' + decodedSubtitle, done);
});
});
@ -115,66 +178,78 @@ function downloadSubtitle(config: IConfig, player: IEpisodePlayer, filePath: str
/**
* Streams the video to disk.
*/
function downloadVideo(config: IConfig,
page: IEpisodePage,
player: IEpisodePlayer,
filePath: string,
done: (err: Error) => void) {
video.stream(
player.video.host,
player.video.file,
page.swf,
filePath, path.extname(player.video.file),
player.video.mode,
done);
function downloadVideo(config: IConfig, page: IEpisodePage, player: IEpisodePlayer,
filePath: string, done: (err: Error) => void)
{
video.stream(player.video.host,player.video.file, page.swf, filePath, path.extname(player.video.file),
player.video.mode, done);
}
/**
* Names the file based on the config, page, series and tag.
*/
function name(config: IConfig, page: IEpisodePage, series: string, extra: string) {
var episodeNum = parseInt(page.episode, 10);
var volumeNum = parseInt(page.volume, 10);
var episode = (episodeNum < 10 ? '0' : '') + page.episode;
var volume = (volumeNum < 10 ? '0' : '') + page.volume;
var tag = config.tag || 'CrunchyRoll';
return series + ' - s' + volume + 'e' + episode +' - [' + tag + ']' + extra;
function name(config: IConfig, page: IEpisodePage, series: string, extra: string)
{
const episodeNum = parseInt(page.episode, 10);
const volumeNum = parseInt(page.volume, 10);
const episode = (episodeNum < 10 ? '0' : '') + page.episode;
const volume = (volumeNum < 10 ? '0' : '') + page.volume;
const tag = config.tag || 'CrunchyRoll';
return series + ' - s' + volume + 'e' + episode + ' - [' + tag + ']' + extra;
}
/**
* Prefixes a value.
*/
function prefix(value: number|string, length: number) {
var valueString = typeof value !== 'string' ? String(value) : value;
while (valueString.length < length) valueString = '0' + valueString;
function prefix(value: number|string, length: number)
{
let valueString = (typeof value !== 'string') ? String(value) : value;
while (valueString.length < length)
{
valueString = '0' + valueString;
}
return valueString;
}
/**
* Requests the page data and scrapes the id, episode, series and swf.
*/
function scrapePage(config: IConfig, address: string, done: (err: Error, page?: IEpisodePage) => void) {
var id = parseInt((address.match(/[0-9]+$/) || ['0'])[0], 10);
if (!id) return done(new Error('Invalid address.'));
request.get(config, address, (err, result) => {
if (err) return done(err);
var $ = cheerio.load(result);
var swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href'));
var regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/;
var look = $('#showmedia_about_media').text();
var seasonTitle = $('span[itemprop="title"]').text();
var data = regexp.exec(look);
function scrapePage(config: IConfig, address: string, done: (err: Error, page?: IEpisodePage) => void)
{
const id = parseInt((address.match(/[0-9]+$/) || ['0'])[0], 10);
if (!id)
{
return done(new Error('Invalid address.'));
}
request.get(config, address, (err, result) =>
{
if (err)
{
return done(err);
}
const $ = cheerio.load(result);
const swf = /^([^?]+)/.exec($('link[rel=video_src]').attr('href'));
const regexp = /\s*([^\n\r\t\f]+)\n?\s*[^0-9]*([0-9][0-9.]*)?,?\n?\s\s*[^0-9]*((PV )?[S0-9][P0-9.]*[a-fA-F]?)/;
const look = $('#showmedia_about_media').text();
const seasonTitle = $('span[itemprop="title"]').text();
const data = regexp.exec(look);
if (!swf || !data)
{
log.warn('Something wrong in the page at '+address+' (data are: '+look+')');
log.warn('Setting Season to 0 and episode to \’0\’...');
log.warn('Something wrong in the page at ' + address + ' (data are: ' + look + ')');
log.warn('Setting Season to 0 and episode to ’0’...');
done(null, {
id: id,
episode: "0",
episode: '0',
series: seasonTitle,
swf: swf[1],
volume: "0"
volume: '0'
});
}
done(null, {
@ -182,7 +257,7 @@ function scrapePage(config: IConfig, address: string, done: (err: Error, page?:
episode: data[3],
series: data[1],
swf: swf[1],
volume: data[2] || "1"
volume: data[2] || '1'
});
});
}
@ -190,26 +265,45 @@ function scrapePage(config: IConfig, address: string, done: (err: Error, page?:
/**
* Requests the player data and scrapes the subtitle and video data.
*/
function scrapePlayer(config: IConfig, address: string, id: number, done: (err: Error, player?: IEpisodePlayer) => void) {
var url = address.match(/^(https?:\/\/[^\/]+)/);
if (!url) return done(new Error('Invalid address.'));
function scrapePlayer(config: IConfig, address: string, id: number, done: (err: Error, player?: IEpisodePlayer) => void)
{
const url = address.match(/^(https?:\/\/[^\/]+)/);
if (!url)
{
return done(new Error('Invalid address.'));
}
request.post(config, {
form: {current_page: address},
url: url[1] + '/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=' + id
}, (err, result) => {
if (err) return done(err);
}, (err, result) =>
{
if (err)
{
return done(err);
}
xml2js.parseString(result, {
explicitArray: false,
explicitRoot: false
}, (err: Error, player: IEpisodePlayerConfig) => {
if (err) return done(err);
try {
var isSubtitled = Boolean(player['default:preload'].subtitle);
var streamMode="RTMP";
if (player['default:preload'].stream_info.host == "")
}, (err: Error, player: IEpisodePlayerConfig) =>
{
if (err)
{
return done(err);
}
try
{
const isSubtitled = Boolean(player['default:preload'].subtitle);
let streamMode = 'RTMP';
if (player['default:preload'].stream_info.host === '')
{
streamMode="HLS";
streamMode = 'HLS';
}
done(null, {
subtitle: isSubtitled ? {
id: parseInt(player['default:preload'].subtitle.$.id, 10),
@ -222,7 +316,8 @@ function scrapePlayer(config: IConfig, address: string, id: number, done: (err:
host: player['default:preload'].stream_info.host
}
});
} catch (parseError) {
} catch (parseError)
{
done(parseError);
}
});

135
src/request.ts
View File

@ -2,11 +2,13 @@
import request = require('request');
import cheerio = require('cheerio');
import log = require('./log');
var cloudscraper = require('cloudscraper');
var isAuthenticated = false;
var isPremium = false;
const cloudscraper = require('cloudscraper');
var defaultHeaders: request.Headers = {
let isAuthenticated = false;
let isPremium = false;
const defaultHeaders: request.Headers =
{
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
'Connection': 'keep-alive'
};
@ -14,11 +16,22 @@ var defaultHeaders: request.Headers = {
/**
* Performs a GET request for the resource.
*/
export function get(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void) {
authenticate(config, err => {
if (err) return done(err);
cloudscraper.request(modify(options, 'GET'), (err: Error, response: any, body: any) => {
if (err) return done(err);
export function get(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void)
{
authenticate(config, err =>
{
if (err)
{
return done(err);
}
cloudscraper.request(modify(options, 'GET'), (err: Error, response: any, body: any) =>
{
if (err)
{
return done(err);
}
done(null, typeof body === 'string' ? body : String(body));
});
});
@ -27,11 +40,22 @@ export function get(config: IConfig, options: request.Options, done: (err: Error
/**
* Performs a POST request for the resource.
*/
export function post(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void) {
authenticate(config, err => {
if (err) return done(err);
cloudscraper.request(modify(options, 'POST'), (err: Error, response: any, body: any) => {
if (err) return done(err);
export function post(config: IConfig, options: request.Options, done: (err: Error, result?: string) => void)
{
authenticate(config, err =>
{
if (err)
{
return done(err);
}
cloudscraper.request(modify(options, 'POST'), (err: Error, response: any, body: any) =>
{
if (err)
{
return done(err);
}
done(null, typeof body === 'string' ? body : String(body));
});
});
@ -40,11 +64,15 @@ export function post(config: IConfig, options: request.Options, done: (err: Erro
/**
* Authenticates using the configured pass and user.
*/
function authenticate(config: IConfig, done: (err: Error) => void) {
if (isAuthenticated || !config.pass || !config.user) return done(null);
function authenticate(config: IConfig, done: (err: Error) => void)
{
if (isAuthenticated || !config.pass || !config.user)
{
return done(null);
}
/* Bypass the login page and send a login request directly */
var options =
let options =
{
headers: defaultHeaders,
jar: true,
@ -53,18 +81,20 @@ function authenticate(config: IConfig, done: (err: Error) => void) {
url: 'https://www.crunchyroll.com/login'
};
// request(options, (err: Error, rep: string, body: string) =>
cloudscraper.request(options, (err: Error, rep: string, body: string) =>
{
if (err) return done(err);
var $ = cheerio.load(body);
const $ = cheerio.load(body);
/* Get the token from the login page */
var token = $('input[name="login_form[_token]"]').attr('value');
if (token === '') return done(new Error('Can`t find token!'));
const token = $('input[name="login_form[_token]"]').attr('value');
if (token === '')
{
return done(new Error('Can`t find token!'));
}
var options =
let options =
{
headers: defaultHeaders,
form:
@ -79,14 +109,18 @@ function authenticate(config: IConfig, done: (err: Error) => void) {
method: 'POST',
url: 'https://www.crunchyroll.com/login'
};
// request.post(options, (err: Error, rep: string, body: string) =>
cloudscraper.request(options, (err: Error, rep: string, body: string) =>
{
if (err) return done(err);
if (err)
{
return done(err);
}
/* The page return with a meta based redirection, as we wan't to check that everything is fine, reload
* the main page. A bit convoluted, but more sure.
*/
var options =
let options =
{
headers: defaultHeaders,
jar: true,
@ -96,23 +130,44 @@ function authenticate(config: IConfig, done: (err: Error) => void) {
cloudscraper.request(options, (err: Error, rep: string, body: string) =>
{
if (err) return done(err);
var $ = cheerio.load(body);
/* Check if auth worked */
var regexps = /ga\(\'set\', \'dimension[5-8]\', \'([^']*)\'\);/g;
var dims = regexps.exec($('script').text());
for (var i = 1; i < 5; i++)
if (err)
{
if ((dims[i] !== undefined) && (dims[i] !== '') && (dims[i] !== 'not-registered')) { isAuthenticated = true; }
if ((dims[i] === 'premium') || (dims[i] === 'premiumplus')) { isPremium = true; }
return done(err);
}
let $ = cheerio.load(body);
/* Check if auth worked */
const regexps = /ga\('set', 'dimension[5-8]', '([^']*)'\);/g;
const dims = regexps.exec($('script').text());
for (let i = 1; i < 5; i++)
{
if ((dims[i] !== undefined) && (dims[i] !== '') && (dims[i] !== 'not-registered'))
{
isAuthenticated = true;
}
if ((dims[i] === 'premium') || (dims[i] === 'premiumplus'))
{
isPremium = true;
}
}
if (isAuthenticated === false)
{
var error = $('ul.message, li.error').text();
const error = $('ul.message, li.error').text();
return done(new Error('Authentication failed: ' + error));
}
if (isPremium === false) { log.warn('Do not use this app without a premium account.'); }
else { log.info('You have a premium account! Good!'); }
if (isPremium === false)
{
log.warn('Do not use this app without a premium account.');
}
else
{
log.info('You have a premium account! Good!');
}
done(null);
});
});
@ -122,12 +177,14 @@ function authenticate(config: IConfig, done: (err: Error) => void) {
/**
* Modifies the options to use the authenticated cookie jar.
*/
function modify(options: string|request.Options, reqMethod: string): request.Options {
if (typeof options !== 'string') {
function modify(options: string|request.Options, reqMethod: string): request.Options
{
if (typeof options !== 'string')
{
options.jar = true;
options.headers = defaultHeaders;
options.method = reqMethod;
return options;
}
return { jar: true, headers: defaultHeaders, url: options.toString(), method: reqMethod};
return { jar: true, headers: defaultHeaders, url: options.toString(), method: reqMethod };
}

139
src/series.ts
View File

@ -6,26 +6,42 @@ import request = require('./request');
import path = require('path');
import url = require('url');
import log = require('./log');
var persistent = '.crpersistent';
const persistent = '.crpersistent';
/**
* Streams the series to disk.
*/
export default function(config: IConfig, address: string, done: (err: Error) => void) {
var persistentPath = path.join(config.output || process.cwd(), persistent);
fs.readFile(persistentPath, 'utf8', (err, contents) => {
var cache = config.cache ? {} : JSON.parse(contents || '{}');
page(config, address, (err, page) => {
if (err) return done(err);
var i = 0;
(function next() {
export default function(config: IConfig, address: string, done: (err: Error) => void)
{
const persistentPath = path.join(config.output || process.cwd(), persistent);
fs.readFile(persistentPath, 'utf8', (err, contents) =>
{
const cache = config.cache ? {} : JSON.parse(contents || '{}');
page(config, address, (err, page) =>
{
if (err)
{
return done(err);
}
let i = 0;
(function next()
{
if (i >= page.episodes.length) return done(null);
download(cache, config, address, page.episodes[i], (err, ignored) => {
if (err) return done(err);
if ((ignored == false) || (ignored == undefined))
download(cache, config, address, page.episodes[i], (err, ignored) =>
{
if (err)
{
var newCache = JSON.stringify(cache, null, ' ');
fs.writeFile(persistentPath, newCache, err => {
return done(err);
}
if ((ignored === false) || (ignored === undefined))
{
const newCache = JSON.stringify(cache, null, ' ');
fs.writeFile(persistentPath, newCache, err =>
{
if (err) return done(err);
i += 1;
next();
@ -45,16 +61,29 @@ export default function(config: IConfig, address: string, done: (err: Error) =>
/**
* Downloads the episode.
*/
function download(cache: {[address: string]: number},
config: IConfig,
baseAddress: string,
item: ISeriesEpisode,
done: (err: Error, ign: boolean) => void) {
if (!filter(config, item)) return done(null, false);
var address = url.resolve(baseAddress, item.address);
if (cache[address]) return done(null, false);
episode(config, address, (err, ignored) => {
if (err) return done(err, false);
function download(cache: {[address: string]: number}, config: IConfig,
baseAddress: string, item: ISeriesEpisode,
done: (err: Error, ign: boolean) => void)
{
if (!filter(config, item))
{
return done(null, false);
}
const address = url.resolve(baseAddress, item.address);
if (cache[address])
{
return done(null, false);
}
episode(config, address, (err, ignored) =>
{
if (err)
{
return done(err, false);
}
cache[address] = Date.now();
done(null, ignored);
});
@ -63,14 +92,17 @@ function download(cache: {[address: string]: number},
/**
* Filters the item based on the configuration.
*/
function filter(config: IConfig, item: ISeriesEpisode) {
function filter(config: IConfig, item: ISeriesEpisode)
{
// Filter on chapter.
var episodeFilter = config.episode;
const episodeFilter = config.episode;
if (episodeFilter > 0 && parseInt(item.episode, 10) <= episodeFilter) return false;
if (episodeFilter < 0 && parseInt(item.episode, 10) >= -episodeFilter) return false;
// Filter on volume.
var volumeFilter = config.volume;
const volumeFilter = config.volume;
if (volumeFilter > 0 && item.volume <= volumeFilter) return false;
if (volumeFilter < 0 && item.volume >= -volumeFilter) return false;
return true;
@ -79,27 +111,50 @@ function filter(config: IConfig, item: ISeriesEpisode) {
/**
* Requests the page and scrapes the episodes and series.
*/
function page(config: IConfig, address: string, done: (err: Error, result?: ISeries) => void) {
request.get(config, address, (err, result) => {
if (err) return done(err);
var $ = cheerio.load(result);
var title = $('span[itemprop=name]').text();
if (!title) return done(new Error('Invalid page.(' + address + ')'));
log.info("Checking availability for " + title);
var episodes: ISeriesEpisode[] = [];
$('.episode').each((i, el) => {
if ($(el).children('img[src*=coming_soon]').length) return;
var volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
var regexp = /Episode\s+((PV )?[S0-9][P0-9.]*[a-fA-F]?)\s*$/i;
var episode = regexp.exec($(el).children('.series-title').text());
var address = $(el).attr('href');
if (!address || !episode) return;
function page(config: IConfig, address: string, done: (err: Error, result?: ISeries) => void)
{
request.get(config, address, (err, result) =>
{
if (err)
{
return done(err);
}
const $ = cheerio.load(result);
const title = $('span[itemprop=name]').text();
if (!title)
{
return done(new Error('Invalid page.(' + address + ')'));
}
log.info('Checking availability for ' + title);
const episodes: ISeriesEpisode[] = [];
$('.episode').each((i, el) =>
{
if ($(el).children('img[src*=coming_soon]').length)
{
return;
}
const volume = /([0-9]+)\s*$/.exec($(el).closest('ul').prev('a').text());
const regexp = /Episode\s+((PV )?[S0-9][P0-9.]*[a-fA-F]?)\s*$/i;
const episode = regexp.exec($(el).children('.series-title').text());
const address = $(el).attr('href');
if (!address || !episode)
{
return;
}
episodes.push({
address: address,
episode: episode[1],
volume: volume ? parseInt(volume[0], 10) : 1
});
});
done(null, {episodes: episodes.reverse(), series: title});
});
}

65
src/subtitle/decode.ts
View File

@ -7,10 +7,14 @@ import zlib = require('zlib');
/**
* Decodes the data.
*/
export default function(id: number, iv: Buffer|string, data: Buffer|string, done: (err?: Error, result?: Buffer) => void) {
try {
export default function(id: number, iv: Buffer|string, data: Buffer|string,
done: (err?: Error, result?: Buffer) => void)
{
try
{
decompress(decrypt(id, iv, data), done);
} catch (e) {
} catch (e)
{
done(e);
}
}
@ -18,21 +22,27 @@ import zlib = require('zlib');
/**
* Decrypts the data.
*/
function decrypt(id: number, iv: Buffer|string, data: Buffer|string) {
var ivBuffer = typeof iv === 'string' ? new Buffer(iv, 'base64') : iv;
var dataBuffer = typeof data === 'string' ? new Buffer(data, 'base64') : data;
var decipher = crypto.createDecipheriv('aes-256-cbc', key(id), ivBuffer);
function decrypt(id: number, iv: Buffer|string, data: Buffer|string)
{
const ivBuffer = typeof iv === 'string' ? new Buffer(iv, 'base64') : iv;
const dataBuffer = typeof data === 'string' ? new Buffer(data, 'base64') : data;
const decipher = crypto.createDecipheriv('aes-256-cbc', key(id), ivBuffer);
decipher.setAutoPadding(false);
return Buffer.concat([decipher.update(dataBuffer), decipher.final()]);
}
/**
* Decompresses the data.
*/
function decompress(data: Buffer, done: (err: Error, result?: Buffer) => void) {
try {
function decompress(data: Buffer, done: (err: Error, result?: Buffer) => void)
{
try
{
zlib.inflate(data, done);
} catch (e) {
} catch (e)
{
done(null, data);
}
}
@ -40,36 +50,45 @@ function decompress(data: Buffer, done: (err: Error, result?: Buffer) => void) {
/**
* Generates a key.
*/
function key(subtitleId: number): Buffer {
var hash = secret(20, 97, 1, 2) + magic(subtitleId);
var result = new Buffer(32);
function key(subtitleId: number): Buffer
{
const hash = secret(20, 97, 1, 2) + magic(subtitleId);
const result = new Buffer(32);
result.fill(0);
crypto.createHash('sha1').update(hash).digest().copy(result);
return result;
}
/**
* Generates a magic number.
*/
function magic(subtitleId: number): number {
var base = Math.floor(Math.sqrt(6.9) * Math.pow(2, 25));
var hash = bigInt(base).xor(subtitleId).toJSNumber();
var multipliedHash = bigInt(hash).multiply(32).toJSNumber();
function magic(subtitleId: number): number
{
const base = Math.floor(Math.sqrt(6.9) * Math.pow(2, 25));
const hash = bigInt(base).xor(subtitleId).toJSNumber();
const multipliedHash = bigInt(hash).multiply(32).toJSNumber();
return bigInt(hash).xor(hash >> 3).xor(multipliedHash).toJSNumber();
}
/**
* Generates a secret string based on a Fibonacci sequence.
*/
function secret(size: number, modulo: number, firstSeed: number, secondSeed: number): string {
var currentValue = firstSeed + secondSeed;
var previousValue = secondSeed;
var result = '';
for (var i = 0; i < size; i += 1) {
var oldValue = currentValue;
function secret(size: number, modulo: number, firstSeed: number, secondSeed: number): string
{
let currentValue = firstSeed + secondSeed;
let previousValue = secondSeed;
let result = '';
for (let i = 0; i < size; i += 1)
{
const oldValue = currentValue;
result += String.fromCharCode(currentValue % modulo + 33);
currentValue += previousValue;
previousValue = oldValue;
}
return result;
}

126
src/subtitle/formats/ass.ts
View File

@ -4,17 +4,25 @@ import xml2js = require('xml2js');
/**
* Converts an input buffer to a SubStation Alpha subtitle.
*/
export default function(input: string|Buffer, done: (err: Error, subtitle?: string) => void) {
export default function(input: string|Buffer, done: (err: Error, subtitle?: string) => void)
{
xml2js.parseString(input.toString(), {
explicitArray: false,
explicitRoot: false
}, (err: Error, xml: ISubtitle) => {
if (err) return done(err);
try {
}, (err: Error, xml: ISubtitle) =>
{
if (err)
{
return done(err);
}
try
{
done(null, script(xml) + '\n' +
style(xml.styles) + '\n' +
event(xml.events));
} catch (err) {
style(xml.styles) + '\n' +
event(xml.events));
} catch (err)
{
done(err);
}
});
@ -23,69 +31,73 @@ export default function(input: string|Buffer, done: (err: Error, subtitle?: stri
/**
* Converts the event block.
*/
function event(block: ISubtitleEvent): string {
function event(block: ISubtitleEvent): string
{
var format = 'Layer,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text';
return '[Events]\n' +
'Format: ' + format + '\n' +
[].concat(block.event).map(style => ('Dialogue: 0,' +
style.$.start + ',' +
style.$.end + ',' +
style.$.style + ',' +
style.$.name + ',' +
style.$.margin_l + ',' +
style.$.margin_r + ',' +
style.$.margin_v + ',' +
style.$.effect + ',' +
style.$.text)).join('\n') + '\n';
'Format: ' + format + '\n' + [].concat(block.event).map(style => ('Dialogue: 0,' +
style.$.start + ',' +
style.$.end + ',' +
style.$.style + ',' +
style.$.name + ',' +
style.$.margin_l + ',' +
style.$.margin_r + ',' +
style.$.margin_v + ',' +
style.$.effect + ',' +
style.$.text)).join('\n') + '\n';
}
/**
* Converts the script block.
*/
function script(block: ISubtitle): string {
function script(block: ISubtitle): string
{
return '[Script Info]\n' +
'Title: ' + block.$.title + '\n' +
'ScriptType: v4.00+\n' +
'WrapStyle: ' + block.$.wrap_style + '\n' +
'PlayResX: ' + block.$.play_res_x + '\n' +
'PlayResY: ' + block.$.play_res_y + '\n' +
'Subtitle ID: ' + block.$.id + '\n' +
'Language: ' + block.$.lang_string + '\n' +
'Created: ' + block.$.created + '\n';
'Title: ' + block.$.title + '\n' +
'ScriptType: v4.00+\n' +
'WrapStyle: ' + block.$.wrap_style + '\n' +
'PlayResX: ' + block.$.play_res_x + '\n' +
'PlayResY: ' + block.$.play_res_y + '\n' +
'Subtitle ID: ' + block.$.id + '\n' +
'Language: ' + block.$.lang_string + '\n' +
'Created: ' + block.$.created + '\n';
}
/**
* Converts the style block.
*/
function style(block: ISubtitleStyle): string {
function style(block: ISubtitleStyle): string
{
var format = 'Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,' +
'OutlineColour,BackColour,Bold,Italic,Underline,StrikeOut,ScaleX,' +
'ScaleY,Spacing,Angle,BorderStyle,Outline,Shadow,Alignment,' +
'MarginL,MarginR,MarginV,Encoding';
'OutlineColour,BackColour,Bold,Italic,Underline,StrikeOut,ScaleX,' +
'ScaleY,Spacing,Angle,BorderStyle,Outline,Shadow,Alignment,' +
'MarginL,MarginR,MarginV,Encoding';
return '[V4+ Styles]\n' +
'Format: ' + format + '\n' +
[].concat(block.style).map(style => 'Style: ' +
style.$.name + ',' +
style.$.font_name + ',' +
style.$.font_size + ',' +
style.$.primary_colour + ',' +
style.$.secondary_colour + ',' +
style.$.outline_colour + ',' +
style.$.back_colour + ',' +
style.$.bold + ',' +
style.$.italic + ',' +
style.$.underline + ',' +
style.$.strikeout + ',' +
style.$.scale_x + ',' +
style.$.scale_y + ',' +
style.$.spacing + ',' +
style.$.angle + ',' +
style.$.border_style + ',' +
style.$.outline + ',' +
style.$.shadow + ',' +
style.$.alignment + ',' +
style.$.margin_l + ',' +
style.$.margin_r + ',' +
style.$.margin_v + ',' +
style.$.encoding).join('\n') + '\n';
'Format: ' + format + '\n' + [].concat(block.style).map(style => 'Style: ' +
style.$.name + ',' +
style.$.font_name + ',' +
style.$.font_size + ',' +
style.$.primary_colour + ',' +
style.$.secondary_colour + ',' +
style.$.outline_colour + ',' +
style.$.back_colour + ',' +
style.$.bold + ',' +
style.$.italic + ',' +
style.$.underline + ',' +
style.$.strikeout + ',' +
style.$.scale_x + ',' +
style.$.scale_y + ',' +
style.$.spacing + ',' +
style.$.angle + ',' +
style.$.border_style + ',' +
style.$.outline + ',' +
style.$.shadow + ',' +
style.$.alignment + ',' +
style.$.margin_l + ',' +
style.$.margin_r + ',' +
style.$.margin_v + ',' +
style.$.encoding).join('\n') + '\n';
}

86
src/subtitle/formats/srt.ts
View File

@ -4,18 +4,30 @@ import xml2js = require('xml2js');
/**
* Converts an input buffer to a SubRip subtitle.
*/
export default function(input: Buffer|string, done: (err: Error, subtitle?: string) => void) {
var options = {explicitArray: false, explicitRoot: false};
xml2js.parseString(input.toString(), options, (err: Error, xml: ISubtitle) => {
try {
if (err) return done(err);
done(null, xml.events.event.map((event, index) => {
var attributes = event.$;
export default function(input: Buffer|string, done: (err: Error, subtitle?: string) => void)
{
const options = {explicitArray: false, explicitRoot: false};
xml2js.parseString(input.toString(), options, (err: Error, xml: ISubtitle) =>
{
try
{
if (err)
{
return done(err);
}
done(null, xml.events.event.map((event, index) =>
{
const attributes = event.$;
return (index + 1) + '\n' +
time(attributes.start) + ' --> ' + time(attributes.end) + '\n' +
text(attributes.text) + '\n';
time(attributes.start) + ' --> ' + time(attributes.end) + '\n' +
text(attributes.text) + '\n';
}).join('\n'));
} catch (err) {
} catch (err)
{
done(err);
}
});
@ -24,41 +36,59 @@ import xml2js = require('xml2js');
/**
* Prefixes a value.
*/
function prefix(value: string, length: number): string {
while (value.length < length) value = '0' + value;
function prefix(value: string, length: number): string
{
while (value.length < length)
{
value = '0' + value;
}
return value;
}
/**
* Suffixes a value.
*/
function suffix(value: string, length: number): string {
while (value.length < length) value = value + '0';
function suffix(value: string, length: number): string
{
while (value.length < length)
{
value = value + '0';
}
return value;
}
/**
* Formats a text value.
*/
function text(value: string): string {
function text(value: string): string
{
return value
.replace(/{\\i1}/g, '<i>').replace(/{\\i0}/g, '</i>')
.replace(/{\\b1}/g, '<b>').replace(/{\\b0}/g, '</b>')
.replace(/{\\u1}/g, '<u>').replace(/{\\u0}/g, '</u>')
.replace(/{[^}]+}/g, '')
.replace(/(\s+)?\\n(\s+)?/ig, '\n')
.trim();
.replace(/{\\i1}/g, '<i>').replace(/{\\i0}/g, '</i>')
.replace(/{\\b1}/g, '<b>').replace(/{\\b0}/g, '</b>')
.replace(/{\\u1}/g, '<u>').replace(/{\\u0}/g, '</u>')
.replace(/{[^}]+}/g, '')
.replace(/(\s+)?\\n(\s+)?/ig, '\n')
.trim();
}
/**
* Formats a time stamp.
*/
function time(value: string): string {
var all = value.match(/^([0-9]+):([0-9]+):([0-9]+)\.([0-9]+)$/);
if (!all) throw new Error('Invalid time.');
var hours = prefix(all[1], 2);
var minutes = prefix(all[2], 2);
var seconds = prefix(all[3], 2);
var milliseconds = suffix(all[4], 3);
function time(value: string): string
{
const all = value.match(/^([0-9]+):([0-9]+):([0-9]+)\.([0-9]+)$/);
if (!all)
{
throw new Error('Invalid time.');
}
const hours = prefix(all[1], 2);
const minutes = prefix(all[2], 2);
const seconds = prefix(all[3], 2);
const milliseconds = suffix(all[4], 3);
return hours + ':' + minutes + ':' + seconds + ',' + milliseconds;
}

79
src/video/merge.ts
View File

@ -8,36 +8,55 @@ import subtitle from '../subtitle/index';
/**
* Merges the subtitle and video files into a Matroska Multimedia Container.
*/
export default function(config: IConfig, isSubtitled: boolean, rtmpInputPath: string, filePath: string, streamMode: string, done: (err: Error) => void) {
var subtitlePath = filePath + '.' + (subtitle.formats[config.format] ? config.format : 'ass');
var videoPath = filePath;
if (streamMode == "RTMP")
export default function(config: IConfig, isSubtitled: boolean, rtmpInputPath: string, filePath: string,
streamMode: string, done: (err: Error) => void)
{
const subtitlePath = filePath + '.' + (subtitle.formats[config.format] ? config.format : 'ass');
let videoPath = filePath;
if (streamMode === 'RTMP')
{
videoPath += path.extname(rtmpInputPath);
}
else
{
videoPath += ".mp4";
videoPath += '.mp4';
}
childProcess.exec(command() + ' ' +
'-o "' + filePath + '.mkv" ' +
'"' + videoPath + '" ' +
(isSubtitled ? '"' + subtitlePath + '"' : ''), {
maxBuffer: Infinity
}, err => {
if (err) return done(err);
unlink(videoPath, subtitlePath, err => {
if (err) unlinkTimeout(videoPath, subtitlePath, 5000);
done(null);
});
'-o "' + filePath + '.mkv" ' +
'"' + videoPath + '" ' +
(isSubtitled ? '"' + subtitlePath + '"' : ''), {
maxBuffer: Infinity,
}, (err) =>
{
if (err)
{
return done(err);
}
unlink(videoPath, subtitlePath, (errin) =>
{
if (errin)
{
unlinkTimeout(videoPath, subtitlePath, 5000);
}
done(null);
});
});
}
/**
* Determines the command for the operating system.
*/
function command(): string {
if (os.platform() !== 'win32') return 'mkvmerge';
function command(): string
{
if (os.platform() !== 'win32')
{
return 'mkvmerge';
}
return '"' + path.join(__dirname, '../../bin/mkvmerge.exe') + '"';
}
@ -45,9 +64,15 @@ function command(): string {
* Unlinks the video and subtitle.
* @private
*/
function unlink(videoPath: string, subtitlePath: string, done: (err: Error) => void) {
fs.unlink(videoPath, err => {
if (err) return done(err);
function unlink(videoPath: string, subtitlePath: string, done: (err: Error) => void)
{
fs.unlink(videoPath, (err) =>
{
if (err)
{
return done(err);
}
fs.unlink(subtitlePath, done);
});
}
@ -55,10 +80,16 @@ function unlink(videoPath: string, subtitlePath: string, done: (err: Error) => v
/**
* Attempts to unlink the video and subtitle with a timeout between each try.
*/
function unlinkTimeout(videoPath: string, subtitlePath: string, timeout: number) {
setTimeout(() => {
unlink(videoPath, subtitlePath, err => {
if (err) unlinkTimeout(videoPath, subtitlePath, timeout);
function unlinkTimeout(videoPath: string, subtitlePath: string, timeout: number)
{
setTimeout(() =>
{
unlink(videoPath, subtitlePath, (err) =>
{
if (err)
{
unlinkTimeout(videoPath, subtitlePath, timeout);
}
});
}, timeout);
}

32
src/video/stream.ts
View File

@ -7,38 +7,44 @@ import log = require('../log');
/**
* Streams the video to disk.
*/
export default function(rtmpUrl: string, rtmpInputPath: string, swfUrl: string, filePath: string, fileExt: string, mode: string, done: (err: Error) => void) {
if (mode == "RTMP")
export default function(rtmpUrl: string, rtmpInputPath: string, swfUrl: string, filePath: string,
fileExt: string, mode: string, done: (err: Error) => void)
{
if (mode === 'RTMP')
{
childProcess.exec(command("rtmpdump") + ' ' +
childProcess.exec(command('rtmpdump') + ' ' +
'-r "' + rtmpUrl + '" ' +
'-y "' + rtmpInputPath + '" ' +
'-W "' + swfUrl + '" ' +
'-o "' + filePath + fileExt + '"', {
maxBuffer: Infinity
maxBuffer: Infinity,
}, done);
}
else if (mode == "HLS")
else if (mode === 'HLS')
{
//log.debug("Experimental FFMPEG, MAY FAIL!!!");
var cmd=command("ffmpeg") + ' ' +
'-i "' + rtmpInputPath + '" ' +
'-c copy -bsf:a aac_adtstoasc ' +
const cmd = command('ffmpeg') + ' ' +
'-i "' + rtmpInputPath + '" ' +
'-c copy -bsf:a aac_adtstoasc ' +
'"' + filePath + '.mp4"';
childProcess.exec(cmd, {
maxBuffer: Infinity
maxBuffer: Infinity,
}, done);
}
else
{
log.error("No such mode: " + mode);
log.error('No such mode: ' + mode);
}
}
/**
* Determines the command for the operating system.
*/
function command(exe: string): string {
if (os.platform() !== 'win32') return exe;
function command(exe: string): string
{
if (os.platform() !== 'win32')
{
return exe;
}
return '"' + path.join(__dirname, '../../bin/' + exe + '.exe') + '"';
}

9
tsconfig.json
View File

@ -41,13 +41,6 @@
"src/video/index.ts",
"src/video/merge.ts",
"src/video/stream.ts",
"typings/big-integer/big-integer.d.ts",
"typings/cheerio/cheerio.d.ts",
"typings/commander/commander.d.ts",
"typings/form-data/form-data.d.ts",
"typings/mkdirp/mkdirp.d.ts",
"typings/node/node.d.ts",
"typings/request/request.d.ts",
"typings/xml2js/xml2js.d.ts"
"typings/index.d.ts"
]
}

9
tslint.json
View File

@ -1,4 +1,5 @@
{
"extends": "tslint:latest",
"rules": {
"ban": false,
"class-name": true,
@ -12,13 +13,13 @@
"interface-name": true,
"jsdoc-format": true,
"label-position": true,
"label-undefined": true,
"max-line-length": [true, 140],
"member-ordering": [true,
"public-before-private",
"static-before-instance",
"variables-before-functions"
],
"array-type": [true, "array"],
"no-any": false,
"no-arg": true,
"no-bitwise": true,
@ -30,19 +31,14 @@
"trace"
],
"no-construct": true,
"no-constructor-vars": true,
"no-debugger": true,
"no-duplicate-key": true,
"no-duplicate-variable": true,
"no-empty": true,
"no-eval": true,
"no-string-literal": true,
"no-switch-case-fall-through": true,
"no-trailing-comma": true,
"no-trailing-whitespace": true,
"no-unused-expression": true,
"no-unused-variable": true,
"no-unreachable": true,
"no-use-before-declare": false,
"no-var-requires": true,
"one-line": [true,
@ -64,7 +60,6 @@
"property-declaration": "nospace",
"variable-declaration": "nospace"
}],
"use-strict": false,
"variable-name": false,
"whitespace": [true,
"check-branch",