565 lines
17 KiB
JavaScript
Executable File
565 lines
17 KiB
JavaScript
Executable File
const {proxy} = require('pinski/plugins')
|
|
const fs = require('fs')
|
|
const childProcess = require('child_process')
|
|
const {request} = require("../utils/request")
|
|
const {getUser} = require("../utils/getuser")
|
|
const converters = require("../utils/converters")
|
|
const constants = require("../utils/constants")
|
|
const path = require("path")
|
|
const nodeFetch = require("node-fetch")
|
|
const yt2009constants = require("./yt2009constants.json")
|
|
const cacheManager = require("../eirtubeMods/cache-manager")
|
|
const dearrow = require("./sb")
|
|
const qualitySort = require("./quality-sort")
|
|
|
|
const downloadInPartsSize = 9_000_000 // 9 MB
|
|
|
|
const cuda_enabled = childProcess.execSync("ffmpeg -hide_banner -loglevel error -hwaccel cuda -i html/static/images/bow.png bow.png && rm bow.png") == ""
|
|
|
|
function downloadStatus(videoID) {
|
|
let dlData = cacheManager.read("download")[videoID]
|
|
if (!dlData)
|
|
return 0;
|
|
else
|
|
return dlData.status
|
|
}
|
|
|
|
async function waitForDownload(videoID) {
|
|
let dlData = cacheManager.read("download")[videoID]
|
|
if (dlData == undefined)
|
|
return
|
|
|
|
await new Promise(resolve => {
|
|
setInterval(() => {
|
|
if (dlData.status == 3 || (dlData.status == 0 && videoExists(videoID)))
|
|
resolve()
|
|
}, 1000 * 2)
|
|
})
|
|
return
|
|
}
|
|
|
|
function videoExists(id) {
|
|
if (!fs.existsSync(path.join(constants.server_setup.video_dl_path, `${id}.mp4`)))
|
|
return false
|
|
const stats = fs.lstatSync(path.join(constants.server_setup.video_dl_path, `${id}.mp4`))
|
|
return stats.size > 5 || stats.isSymbolicLink()
|
|
}
|
|
|
|
function tryDeleteFromCache(id) {
|
|
function del() {
|
|
for (const file of fs.readdirSync(constants.server_setup.video_dl_path))
|
|
if (fs.existsSync(path.join(constants.server_setup.video_dl_path, id)) && !fs.lstatSync(path.join(constants.server_setup.video_dl_path, id)).isDirectory() && file.startsWith(id))
|
|
fs.unlinkSync(path.join(constants.server_setup.video_dl_path, file))
|
|
}
|
|
|
|
if (downloadStatus(id) > 0)
|
|
waitForDownload(id).then(del)
|
|
else
|
|
del()
|
|
}
|
|
|
|
// This method is pretty much STOLEN from yt2009. thank you :)
|
|
async function saveMp4Android(id, quality) {
|
|
if (quality == undefined)
|
|
quality = "360p"
|
|
|
|
let fname = `${id}-${quality}`
|
|
const originalFName = fname
|
|
|
|
// Check if file is already downloading
|
|
let dlData = cacheManager.read("download")[fname]
|
|
|
|
// Account for quality by checking for fname instead of id
|
|
if (videoExists(fname)) {
|
|
if (dlData != undefined && dlData.status == 2)
|
|
await waitForDownload(fname)
|
|
dlData.status = 3
|
|
return
|
|
} else
|
|
dlData = {}
|
|
|
|
dlData.status = 2 // Starting download
|
|
cacheManager.write("download", fname, dlData)
|
|
|
|
// Copies a dict i think
|
|
let headers = JSON.parse(JSON.stringify(yt2009constants.headers))
|
|
headers["user-agent"] = "com.google.android.youtube/19.02.39 (Linux; U; Android 14) gzip"
|
|
|
|
let response = await nodeFetch("https://www.youtube.com/youtubei/v1/player?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", {
|
|
headers: headers,
|
|
referrer: "https://www.youtube.com/watch?v=" + id,
|
|
referrerPolicy: "origin-when-cross-origin",
|
|
body: JSON.stringify({
|
|
context: {
|
|
"client": {
|
|
"hl": "en",
|
|
"clientName": "ANDROID",
|
|
"clientVersion": "19.02.39",
|
|
"androidSdkVersion": 34,
|
|
"mainAppWebInfo": {
|
|
"graftUrl": "/watch?v=" + id
|
|
}
|
|
}
|
|
},
|
|
videoId: id,
|
|
racyCheckOk: true,
|
|
contentCheckOk: true
|
|
}),
|
|
method: "POST",
|
|
mode: "cors"
|
|
})
|
|
|
|
let r
|
|
try {
|
|
r = await response.json()
|
|
} catch (e) {
|
|
if (e instanceof fetch.FetchError)
|
|
console.log("FetchError while accessing /youtubei/v1/player:\n" + await response.text())
|
|
r = {}
|
|
}
|
|
|
|
// Fail state
|
|
if (!r.streamingData || !r.streamingData.formats || r.streamingData.formats.length == 0) {
|
|
dlData.status = 3
|
|
cacheManager.write("download", fname, dlData)
|
|
return
|
|
}
|
|
|
|
let qualities = {}
|
|
let h264DashAudioUrl;
|
|
// prefer nondash formats
|
|
r.streamingData.formats.forEach(q => {
|
|
q.dash = false;
|
|
qualities[q.qualityLabel] = q;
|
|
})
|
|
// add h264 dash formats
|
|
let audioFormats = []
|
|
let aidub = false;
|
|
r.streamingData.adaptiveFormats.forEach(q => {
|
|
if (q.mimeType.includes("audio/") && q.resolution && q.resolution.includes("default"))
|
|
aidub = true;
|
|
if(q.mimeType.includes("audio/mp4")) {
|
|
audioFormats.push(q)
|
|
} else if(q.mimeType.includes("video/mp4")
|
|
&& q.mimeType.includes("avc")
|
|
&& !qualities[q.qualityLabel]) {
|
|
q.dash = true;
|
|
qualities[q.qualityLabel] = q;
|
|
}
|
|
})
|
|
if(audioFormats.length > 0) {
|
|
audioFormats = audioFormats.sort((a, b) => {
|
|
if (aidub) {
|
|
const a_is_default = a.resolution && a.resolution.includes("default");
|
|
const b_is_default = b.resolution && b.resolution.includes("default");
|
|
|
|
if (a_is_default || b_is_default)
|
|
return ((b_is_default ? 1000000 : 0) + b.bitrate) - ((a_is_default ? 1000000 : 0) + a.bitrate);
|
|
}
|
|
return b.bitrate - a.bitrate
|
|
})
|
|
}
|
|
h264DashAudioUrl = audioFormats[0].url
|
|
|
|
// check if dash audio is needed
|
|
// we can pull from already download mp4 if not
|
|
let downloadAudio = true;
|
|
let audioDownloadDone = false;
|
|
let videoDownloadDone = false;
|
|
/*if(fs.existsSync("../assets/" + id + ".mp4")) {
|
|
downloadAudio = false;
|
|
}
|
|
if(quality == "360p" && !qualities["360p"])
|
|
for(let q in qualities) {
|
|
if(qualities[q].itag == 18) {
|
|
quality = q;
|
|
break;
|
|
}
|
|
}
|
|
*/
|
|
if(qualities[quality] && !qualities[quality].dash)
|
|
downloadAudio = false;
|
|
|
|
// Get best available quality
|
|
quality = qualitySort.sort(Object.keys(qualities), quality)[0]
|
|
fname = `${id}-${quality}`
|
|
|
|
// Does file with new name already exist?
|
|
if (originalFName != fname && fs.existsSync(path.join(constants.server_setup.video_dl_path, `${fname}.mp4`))) {
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
if (!fs.existsSync(path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`)))
|
|
fs.symlinkSync(`${fname}.mp4`, path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`), "file")
|
|
return //path.join(constants.server_setup.video_dl_path, `${fname}.mp4`)
|
|
}
|
|
|
|
let finalReturn
|
|
|
|
if(downloadAudio) {
|
|
downloadInParts_file(h264DashAudioUrl, path.join(constants.server_setup.video_dl_path, `${fname}-audio.m4a`))
|
|
.then(() => {
|
|
audioDownloadDone = true;
|
|
if(audioDownloadDone && videoDownloadDone)
|
|
onFormatsDone()
|
|
})
|
|
} else {
|
|
// Use ytdlp as a backup
|
|
const result = await downloadInParts_file(qualities[quality].url, path.join(constants.server_setup.video_dl_path, `${fname}.mp4`))
|
|
if (!result) {
|
|
let out = path.join(constants.server_setup.video_dl_path, `${fname}.mp4`)
|
|
fs.unlinkSync(out)
|
|
|
|
// Brief pause to let file delete finish
|
|
await new Promise(resolve => setTimeout(resolve, 1000))
|
|
|
|
await downloadWithYtdlp(id, qualities[quality].itag, out)
|
|
/*
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
return out
|
|
*/
|
|
}
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
if (originalFName != fname)
|
|
fs.symlinkSync(`${fname}.mp4`, path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`), "file")
|
|
return
|
|
}
|
|
|
|
if(!qualities[quality]) {
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
return //false;
|
|
}
|
|
|
|
// Use ytdlp as a backup
|
|
const result = await downloadInParts_file(qualities[quality].url, path.join(constants.server_setup.video_dl_path, `${id}-temp-${quality}.mp4`))
|
|
if (!result) {
|
|
let out = path.join(constants.server_setup.video_dl_path, `${id}-temp-${quality}.mp4`)
|
|
fs.unlinkSync(out)
|
|
|
|
// Brief pause to let file delete finish
|
|
await new Promise(resolve => setTimeout(resolve, 1000))
|
|
|
|
await downloadWithYtdlp(id, qualities[quality].itag, out)
|
|
/*
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
return out
|
|
*/
|
|
}
|
|
videoDownloadDone = true;
|
|
if(audioDownloadDone || !downloadAudio) {
|
|
onFormatsDone()
|
|
}
|
|
|
|
// merge formats once both are ready
|
|
function onFormatsDone() {
|
|
const pathBase = path.join(__dirname, "../")
|
|
|
|
let audioPath = path.join(constants.server_setup.video_dl_path, `${fname}-audio.m4a`)
|
|
if(!downloadAudio) {
|
|
audioPath = path.join(constants.server_setup.video_dl_path, `${fname}.mp4`)
|
|
}
|
|
let videoPath = path.join(constants.server_setup.video_dl_path, `${id}-temp-${quality}.mp4`)
|
|
|
|
let cmd = [
|
|
"ffmpeg",
|
|
cuda_enabled ? "-hwaccel cuda -c:v h264_cuvid" : null,
|
|
`-y -i ${path.join(pathBase, videoPath)}`,
|
|
`-i ${path.join(pathBase, audioPath)}`,
|
|
`-c:v copy -c:a copy`,
|
|
`-map 0:v -map 1:a`,
|
|
path.join(pathBase, path.join(constants.server_setup.video_dl_path, `${fname}.mp4`))
|
|
].join(" ")
|
|
|
|
childProcess.exec(cmd, { maxBuffer: 1024*1024*1024*5 }, (e) => {
|
|
if(e) {
|
|
console.error(`FFmpeg error while combining streams:\n${e}`)
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
finalReturn = false
|
|
return
|
|
}
|
|
|
|
const videoPathAge = fs.statSync(videoPath).atimeMs
|
|
const audioPathAge = fs.statSync(audioPath).atimeMs
|
|
|
|
setTimeout(() => {
|
|
// delete temp assets
|
|
try {
|
|
if(!audioPath.includes(".mp4")) {
|
|
if (fs.statSync(audioPath).atimeMs == audioPathAge)
|
|
fs.unlinkSync(path.join(pathBase, audioPath))
|
|
}
|
|
if (fs.statSync(videoPath).atimeMs == videoPathAge)
|
|
fs.unlinkSync(path.join(pathBase, videoPath))
|
|
}
|
|
catch(error) {}
|
|
}, 500)
|
|
dlData.status = 3
|
|
cacheManager.write("download", originalFName, dlData)
|
|
cacheManager.write("download", fname, dlData)
|
|
if (originalFName != fname)
|
|
fs.symlinkSync(`${fname}.mp4`, path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`), "file")
|
|
finalReturn = true
|
|
})
|
|
}
|
|
|
|
await new Promise((resolve) => {
|
|
let f
|
|
let tries = 3
|
|
f = function() {
|
|
if (finalReturn == undefined)
|
|
setTimeout(f, 1000 * 1.5)
|
|
else {
|
|
if (f == false && tries > 0) {
|
|
finalReturn = undefined
|
|
tries--
|
|
onFormatsDone()
|
|
setTimeout(f, 1000 * 1.5)
|
|
} else
|
|
resolve()
|
|
}
|
|
}
|
|
f()
|
|
})
|
|
|
|
return //finalReturn
|
|
}
|
|
|
|
async function downloadInParts_file(url, out) {
|
|
let stream = fs.createWriteStream(out, { flags: "a" })
|
|
return await new Promise(resolve => {
|
|
function fetchNextPart(partNumber) {
|
|
let partStartB = partNumber * downloadInPartsSize;
|
|
if(partNumber !== 0) {partStartB += partNumber}
|
|
const newHeaders = { ...yt2009constants.androidHeaders };
|
|
newHeaders.headers.range = `bytes=${partStartB}-${partStartB + downloadInPartsSize}`;
|
|
nodeFetch(url, newHeaders).then(r => {
|
|
// If request returns 403, cancel...
|
|
if (r.status == 403) {
|
|
resolve(false)
|
|
return
|
|
} else if (r.headers.get('Content-Length') === '0') {
|
|
stream.end();
|
|
resolve(true)
|
|
return
|
|
}
|
|
|
|
r.body.pipe(stream, { end: false });
|
|
r.body.on('end', () => {
|
|
fetchNextPart(partNumber + 1);
|
|
})
|
|
})
|
|
}
|
|
fetchNextPart(0)
|
|
})
|
|
}
|
|
|
|
async function downloadWithYtdlp(id, itag, out) {
|
|
await new Promise(resolve => {
|
|
childProcess.exec(`yt-dlp -f ${itag} --cache-dir "${constants.server_setup.ytdlp_cache_path}" --no-part -o "${out}" "${id}"`, e => {
|
|
if (e) {
|
|
console.error(`FFmpeg error while downloading ${id}:\n${e}`)
|
|
resolve({ error: e })
|
|
}
|
|
|
|
resolve({ error: false })
|
|
})
|
|
})
|
|
}
|
|
|
|
async function createFfmpegOgg(videoID, pathIn, pathOut) {
|
|
if (fs.existsSync(pathOut))
|
|
fs.unlinkSync(pathOut)
|
|
|
|
cacheManager.write("ogg", videoID, { status: 1 })
|
|
|
|
return await new Promise(resolve => {
|
|
childProcess.exec(`ffmpeg ${cuda_enabled ? "-hwaccel cuda -c:v h264_cuvid ": ""}-i ${pathIn} -b 1500k -ab 128000 -speed 2 -vn ${pathOut}`, e => {
|
|
cacheManager.write("ogg", videoID, { status: 2 })
|
|
|
|
if (e) {
|
|
console.error(`FFmpeg error while creating ogg:\n${e}`)
|
|
resolve({ error: e })
|
|
}
|
|
|
|
resolve({ error: false })
|
|
})
|
|
})
|
|
}
|
|
|
|
function createThumbStream(key, inFile, outFile, vidLength) {
|
|
let data = cacheManager.read("download")[key] // Use download cache for parity with /getVideo endpoint
|
|
|
|
if (data.status == 2 || videoExists(outFile))
|
|
return
|
|
else {
|
|
data.status = 2
|
|
cacheManager.write("download", key, data)
|
|
|
|
// const vidLength = Number(childProcess.execSync(`ffprobe -i ${inFile}.mp4 -show_entries format=duration -v quiet -of csv="p=0"`))
|
|
let fps = 1
|
|
if (vidLength < 2 * 60)
|
|
fps = 1
|
|
else if (vidLength < 5 * 60)
|
|
fps = 0.5
|
|
else if (vidLength < 15 * 60)
|
|
fps = 0.2
|
|
else
|
|
fps = 0.1
|
|
|
|
const cmd = `nice -n 0 ffmpeg ${cuda_enabled ? "-hwaccel cuda ": ""}-i "${inFile}.mp4" -an -vf "fps=${fps},scale=160:90:flags=lanczos" "${outFile}.mp4"`
|
|
childProcess.exec(cmd, e => {
|
|
data.status = 3
|
|
cacheManager.write("download", key, data)
|
|
|
|
if (e)
|
|
console.error(`FFmpeg error while creating thumb stream:\n${e}`)
|
|
})
|
|
}
|
|
}
|
|
|
|
function formatOrder(format) {
|
|
// most significant to least significant
|
|
// key, max, order, transform
|
|
// asc: lower number comes first, desc: higher number comes first
|
|
const spec = [
|
|
{key: "second__height", max: 8000, order: "desc", transform: x => x ? Math.floor(x/96) : 0},
|
|
{key: "fps", max: 100, order: "desc", transform: x => x ? Math.floor(x/10) : 0},
|
|
{key: "type", max: " ".repeat(60), order: "asc", transform: x => x.length},
|
|
{key: "container", max: "undefined", order: "desc", transform: x => x ? (x == "mp4" ? -1 : 0) : 1}
|
|
]
|
|
let total = 0
|
|
for (let i = 0; i < spec.length; i++) {
|
|
const s = spec[i]
|
|
let diff = s.transform(format[s.key])
|
|
if (s.order === "asc") diff = s.transform(s.max) - diff
|
|
total += diff
|
|
if (i+1 < spec.length) { // not the last spec item?
|
|
const s2 = spec[i+1]
|
|
total *= s2.transform(s2.max)
|
|
}
|
|
}
|
|
return -total
|
|
}
|
|
|
|
module.exports = {
|
|
downloadStatus,
|
|
waitForDownload,
|
|
videoExists,
|
|
tryDeleteFromCache,
|
|
createFfmpegOgg,
|
|
createThumbStream,
|
|
|
|
// Stolen from yt2009
|
|
saveMp4Android,
|
|
fetchVideoData: async function(id, req) {
|
|
// Check if video data already in cache
|
|
let videoData = cacheManager.read("video")[id]
|
|
|
|
if (videoData)
|
|
return videoData
|
|
else {
|
|
// Get video data and set to cache
|
|
// Uses Cloudtube's api usage stuff
|
|
const user = getUser(req)
|
|
const settings = user.getSettingsOrDefaults()
|
|
var instanceOrigin = settings.instance
|
|
var outURL = `${instanceOrigin}/api/v1/videos/${id}`
|
|
|
|
let videoFuture = request(outURL)
|
|
videoData = await videoFuture
|
|
if (!videoData.ok)
|
|
return videoData.result
|
|
videoData = await videoData.result
|
|
|
|
if (videoData.error)
|
|
return videoData
|
|
|
|
videoData = await module.exports.formatVideoData(videoData, settings)
|
|
|
|
return videoData
|
|
}
|
|
},
|
|
formatVideoData: async function(videoData, settings) {
|
|
videoData.formats = await module.exports.sortFormats(videoData, ([480, 720, 1080])[settings.quality])
|
|
// process length text and view count
|
|
for (const rec of videoData.recommendedVideos)
|
|
converters.normaliseVideoInfo(rec)
|
|
// normalise view count
|
|
if (!videoData.second__viewCountText && videoData.viewCount)
|
|
videoData.second__viewCountText = converters.viewCountToText(videoData.viewCount)
|
|
// normalize like count
|
|
if (!videoData.second__likeCountText && videoData.likeCount)
|
|
videoData.second__likeCountText = converters.likeCountToText(videoData.likeCount)
|
|
// rewrite description
|
|
videoData.descriptionHtml = converters.rewriteVideoDescription(videoData.descriptionHtml, videoData.videoId)
|
|
|
|
// rewrite captions urls so they are served on the same domain via the /proxy route
|
|
for (const caption of videoData.captions)
|
|
caption.url = `/getCaption?${new URLSearchParams({ url: caption.url })}`
|
|
|
|
// Apply SponsorBlock data
|
|
if (settings.sponsorblock > 0) {
|
|
const sbData = await dearrow.getSB(videoData.videoId)
|
|
if (sbData && !sbData.error)
|
|
videoData.sbData = sbData.data
|
|
}
|
|
// Apply DeArrow data
|
|
if (settings.dearrow > 0) {
|
|
const dearrowData = await dearrow.getDeArrow(videoData.videoId)
|
|
if (dearrowData && !dearrowData.error)
|
|
videoData.dearrowData = dearrowData.data
|
|
}
|
|
|
|
return videoData
|
|
},
|
|
|
|
sortFormats: async function(video, targetQuality) {
|
|
// if (video.formatStreams == undefined || video.formatStreams.length == 0)
|
|
// return []
|
|
|
|
// Add second__ extensions to format objects, required if Invidious was the extractor
|
|
let formats = (video.formatStreams || []).concat(video.adaptiveFormats)
|
|
for (const format of formats) {
|
|
if (!format.second__height && format.resolution) format.second__height = +format.resolution.slice(0, -1)
|
|
if (!format.second__order) format.second__order = formatOrder(format)
|
|
if (!format.container) format.container = format.type.split("video/")[1].split(";")[0]
|
|
format.cloudtube__label = `${format.qualityLabel} ${format.container}`
|
|
if (!format.clen) {
|
|
try {
|
|
const req = await nodeFetch(format.url, { method: "HEAD" })
|
|
format.clen = req.headers.get('content-length')
|
|
} catch(e) {
|
|
console.error(e)
|
|
}
|
|
}
|
|
if (format.clen)
|
|
format.eirtube__size = `${converters.bytesToSizeText(format.clen)}`
|
|
}
|
|
|
|
// Properly build and order format list
|
|
let standard = video.formatStreams ? video.formatStreams.slice().sort((a, b) => (b.second__height + (b.container == "mp4" ? -1 : 0)) - (a.second__height + (a.container == "mp4" ? -1 : 0))) : []
|
|
let adaptive = video.adaptiveFormats.filter(f => f.type.startsWith("video") && f.qualityLabel).sort((a, b) => a.second__order - b.second__order)
|
|
for (const format of adaptive) {
|
|
if (!format.cloudtube__label.endsWith("*")) format.cloudtube__label += " *"
|
|
}
|
|
|
|
// Sort both groups separately
|
|
standard = qualitySort.sortFormats(standard, targetQuality, 30)
|
|
adaptive = qualitySort.sortFormats(adaptive, targetQuality, 30)
|
|
|
|
formats = standard.concat(adaptive)
|
|
|
|
return formats;
|
|
}
|
|
}
|