const {proxy} = require('pinski/plugins') const fs = require('fs') const childProcess = require('child_process') const {request} = require("../utils/request") const {getUser} = require("../utils/getuser") const converters = require("../utils/converters") const constants = require("../utils/constants") const path = require("path") const nodeFetch = require("node-fetch") const yt2009constants = require("./yt2009constants.json") const cacheManager = require("../eirtubeMods/cache-manager") const dearrow = require("./sb") const qualitySort = require("./quality-sort") const downloadInPartsSize = 9_000_000 // 9 MB const cuda_enabled = childProcess.execSync("ffmpeg -hide_banner -loglevel error -hwaccel cuda -i html/static/images/bow.png bow.png && rm bow.png") == "" function downloadStatus(videoID) { let dlData = cacheManager.read("download")[videoID] if (!dlData) return 0; else return dlData.status } async function waitForDownload(videoID) { let dlData = cacheManager.read("download")[videoID] if (dlData == undefined) return await new Promise(resolve => { setInterval(() => { if (dlData.status == 3 || (dlData.status == 0 && videoExists(videoID))) resolve() }, 1000 * 2) }) return } function videoExists(id) { if (!fs.existsSync(path.join(constants.server_setup.video_dl_path, `${id}.mp4`))) return false const stats = fs.lstatSync(path.join(constants.server_setup.video_dl_path, `${id}.mp4`)) return stats.size > 5 || stats.isSymbolicLink() } function tryDeleteFromCache(id) { function del() { for (const file of fs.readdirSync(constants.server_setup.video_dl_path)) if (fs.existsSync(path.join(constants.server_setup.video_dl_path, id)) && !fs.lstatSync(path.join(constants.server_setup.video_dl_path, id)).isDirectory() && file.startsWith(id)) fs.unlinkSync(path.join(constants.server_setup.video_dl_path, file)) } if (downloadStatus(id) > 0) waitForDownload(id).then(del) else del() } // This method is pretty much STOLEN from yt2009. thank you :) async function saveMp4Android(id, quality) { if (quality == undefined) quality = "360p" let fname = `${id}-${quality}` const originalFName = fname // Check if file is already downloading let dlData = cacheManager.read("download")[fname] // Account for quality by checking for fname instead of id if (videoExists(fname)) { if (dlData != undefined && dlData.status == 2) await waitForDownload(fname) dlData.status = 3 return } else dlData = {} dlData.status = 2 // Starting download cacheManager.write("download", fname, dlData) // Copies a dict i think let headers = JSON.parse(JSON.stringify(yt2009constants.headers)) headers["user-agent"] = "com.google.android.youtube/19.02.39 (Linux; U; Android 14) gzip" let response = await nodeFetch("https://www.youtube.com/youtubei/v1/player?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", { headers: headers, referrer: "https://www.youtube.com/watch?v=" + id, referrerPolicy: "origin-when-cross-origin", body: JSON.stringify({ context: { "client": { "hl": "en", "clientName": "ANDROID", "clientVersion": "19.02.39", "androidSdkVersion": 34, "mainAppWebInfo": { "graftUrl": "/watch?v=" + id } } }, videoId: id, racyCheckOk: true, contentCheckOk: true }), method: "POST", mode: "cors" }) let r try { r = await response.json() } catch (e) { if (e instanceof fetch.FetchError) console.log("FetchError while accessing /youtubei/v1/player:\n" + await response.text()) r = {} } // Fail state if (!r.streamingData || !r.streamingData.formats || r.streamingData.formats.length == 0) { dlData.status = 3 cacheManager.write("download", fname, dlData) return } let qualities = {} let h264DashAudioUrl; // prefer nondash formats r.streamingData.formats.forEach(q => { q.dash = false; qualities[q.qualityLabel] = q; }) // add h264 dash formats let audioFormats = [] let aidub = false; r.streamingData.adaptiveFormats.forEach(q => { if (q.mimeType.includes("audio/") && q.resolution && q.resolution.includes("default")) aidub = true; if(q.mimeType.includes("audio/mp4")) { audioFormats.push(q) } else if(q.mimeType.includes("video/mp4") && q.mimeType.includes("avc") && !qualities[q.qualityLabel]) { q.dash = true; qualities[q.qualityLabel] = q; } }) if(audioFormats.length > 0) { audioFormats = audioFormats.sort((a, b) => { if (aidub) { const a_is_default = a.resolution && a.resolution.includes("default"); const b_is_default = b.resolution && b.resolution.includes("default"); if (a_is_default || b_is_default) return ((b_is_default ? 1000000 : 0) + b.bitrate) - ((a_is_default ? 1000000 : 0) + a.bitrate); } return b.bitrate - a.bitrate }) } h264DashAudioUrl = audioFormats[0].url // check if dash audio is needed // we can pull from already download mp4 if not let downloadAudio = true; let audioDownloadDone = false; let videoDownloadDone = false; /*if(fs.existsSync("../assets/" + id + ".mp4")) { downloadAudio = false; } if(quality == "360p" && !qualities["360p"]) for(let q in qualities) { if(qualities[q].itag == 18) { quality = q; break; } } */ if(qualities[quality] && !qualities[quality].dash) downloadAudio = false; // Get best available quality quality = qualitySort.sort(Object.keys(qualities), quality)[0] fname = `${id}-${quality}` // Does file with new name already exist? if (originalFName != fname && fs.existsSync(path.join(constants.server_setup.video_dl_path, `${fname}.mp4`))) { dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) if (!fs.existsSync(path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`))) fs.symlinkSync(`${fname}.mp4`, path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`), "file") return //path.join(constants.server_setup.video_dl_path, `${fname}.mp4`) } let finalReturn if(downloadAudio) { downloadInParts_file(h264DashAudioUrl, path.join(constants.server_setup.video_dl_path, `${fname}-audio.m4a`)) .then(() => { audioDownloadDone = true; if(audioDownloadDone && videoDownloadDone) onFormatsDone() }) } else { // Use ytdlp as a backup const result = await downloadInParts_file(qualities[quality].url, path.join(constants.server_setup.video_dl_path, `${fname}.mp4`)) if (!result) { let out = path.join(constants.server_setup.video_dl_path, `${fname}.mp4`) fs.unlinkSync(out) // Brief pause to let file delete finish await new Promise(resolve => setTimeout(resolve, 1000)) await downloadWithYtdlp(id, qualities[quality].itag, out) /* dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) return out */ } dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) if (originalFName != fname) fs.symlinkSync(`${fname}.mp4`, path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`), "file") return } if(!qualities[quality]) { dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) return //false; } // Use ytdlp as a backup const result = await downloadInParts_file(qualities[quality].url, path.join(constants.server_setup.video_dl_path, `${id}-temp-${quality}.mp4`)) if (!result) { let out = path.join(constants.server_setup.video_dl_path, `${id}-temp-${quality}.mp4`) fs.unlinkSync(out) // Brief pause to let file delete finish await new Promise(resolve => setTimeout(resolve, 1000)) await downloadWithYtdlp(id, qualities[quality].itag, out) /* dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) return out */ } videoDownloadDone = true; if(audioDownloadDone || !downloadAudio) { onFormatsDone() } // merge formats once both are ready function onFormatsDone() { const pathBase = path.join(__dirname, "../") let audioPath = path.join(constants.server_setup.video_dl_path, `${fname}-audio.m4a`) if(!downloadAudio) { audioPath = path.join(constants.server_setup.video_dl_path, `${fname}.mp4`) } let videoPath = path.join(constants.server_setup.video_dl_path, `${id}-temp-${quality}.mp4`) let cmd = [ "ffmpeg", cuda_enabled ? "-hwaccel cuda -c:v h264_cuvid" : null, `-y -i ${path.join(pathBase, videoPath)}`, `-i ${path.join(pathBase, audioPath)}`, `-c:v copy -c:a copy`, `-map 0:v -map 1:a`, path.join(pathBase, path.join(constants.server_setup.video_dl_path, `${fname}.mp4`)) ].join(" ") childProcess.exec(cmd, { maxBuffer: 1024*1024*1024*5 }, (e) => { if(e) { console.error(`FFmpeg error while combining streams:\n${e}`) dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) finalReturn = false return } const videoPathAge = fs.statSync(videoPath).atimeMs const audioPathAge = fs.statSync(audioPath).atimeMs setTimeout(() => { // delete temp assets try { if(!audioPath.includes(".mp4")) { if (fs.statSync(audioPath).atimeMs == audioPathAge) fs.unlinkSync(path.join(pathBase, audioPath)) } if (fs.statSync(videoPath).atimeMs == videoPathAge) fs.unlinkSync(path.join(pathBase, videoPath)) } catch(error) {} }, 500) dlData.status = 3 cacheManager.write("download", originalFName, dlData) cacheManager.write("download", fname, dlData) if (originalFName != fname) fs.symlinkSync(`${fname}.mp4`, path.join(constants.server_setup.video_dl_path, `${originalFName}.mp4`), "file") finalReturn = true }) } await new Promise((resolve) => { let f let tries = 3 f = function() { if (finalReturn == undefined) setTimeout(f, 1000 * 1.5) else { if (f == false && tries > 0) { finalReturn = undefined tries-- onFormatsDone() setTimeout(f, 1000 * 1.5) } else resolve() } } f() }) return //finalReturn } async function downloadInParts_file(url, out) { let stream = fs.createWriteStream(out, { flags: "a" }) return await new Promise(resolve => { function fetchNextPart(partNumber) { let partStartB = partNumber * downloadInPartsSize; if(partNumber !== 0) {partStartB += partNumber} const newHeaders = { ...yt2009constants.androidHeaders }; newHeaders.headers.range = `bytes=${partStartB}-${partStartB + downloadInPartsSize}`; nodeFetch(url, newHeaders).then(r => { // If request returns 403, cancel... if (r.status == 403) { resolve(false) return } else if (r.headers.get('Content-Length') === '0') { stream.end(); resolve(true) return } r.body.pipe(stream, { end: false }); r.body.on('end', () => { fetchNextPart(partNumber + 1); }) }) } fetchNextPart(0) }) } async function downloadWithYtdlp(id, itag, out) { await new Promise(resolve => { childProcess.exec(`yt-dlp -f ${itag} --cache-dir "${constants.server_setup.ytdlp_cache_path}" --no-part -o "${out}" "${id}"`, e => { if (e) { console.error(`FFmpeg error while downloading ${id}:\n${e}`) resolve({ error: e }) } resolve({ error: false }) }) }) } async function createFfmpegOgg(videoID, pathIn, pathOut) { if (fs.existsSync(pathOut)) fs.unlinkSync(pathOut) cacheManager.write("ogg", videoID, { status: 1 }) return await new Promise(resolve => { childProcess.exec(`ffmpeg ${cuda_enabled ? "-hwaccel cuda -c:v h264_cuvid ": ""}-i ${pathIn} -b 1500k -ab 128000 -speed 2 -vn ${pathOut}`, e => { cacheManager.write("ogg", videoID, { status: 2 }) if (e) { console.error(`FFmpeg error while creating ogg:\n${e}`) resolve({ error: e }) } resolve({ error: false }) }) }) } function createThumbStream(key, inFile, outFile, vidLength) { let data = cacheManager.read("download")[key] // Use download cache for parity with /getVideo endpoint if (data.status == 2 || videoExists(outFile)) return else { data.status = 2 cacheManager.write("download", key, data) // const vidLength = Number(childProcess.execSync(`ffprobe -i ${inFile}.mp4 -show_entries format=duration -v quiet -of csv="p=0"`)) let fps = 1 if (vidLength < 2 * 60) fps = 1 else if (vidLength < 5 * 60) fps = 0.5 else if (vidLength < 15 * 60) fps = 0.2 else fps = 0.1 const cmd = `nice -n 0 ffmpeg ${cuda_enabled ? "-hwaccel cuda ": ""}-i "${inFile}.mp4" -an -vf "fps=${fps},scale=160:90:flags=lanczos" "${outFile}.mp4"` childProcess.exec(cmd, e => { data.status = 3 cacheManager.write("download", key, data) if (e) console.error(`FFmpeg error while creating thumb stream:\n${e}`) }) } } function formatOrder(format) { // most significant to least significant // key, max, order, transform // asc: lower number comes first, desc: higher number comes first const spec = [ {key: "second__height", max: 8000, order: "desc", transform: x => x ? Math.floor(x/96) : 0}, {key: "fps", max: 100, order: "desc", transform: x => x ? Math.floor(x/10) : 0}, {key: "type", max: " ".repeat(60), order: "asc", transform: x => x.length}, {key: "container", max: "undefined", order: "desc", transform: x => x ? (x == "mp4" ? -1 : 0) : 1} ] let total = 0 for (let i = 0; i < spec.length; i++) { const s = spec[i] let diff = s.transform(format[s.key]) if (s.order === "asc") diff = s.transform(s.max) - diff total += diff if (i+1 < spec.length) { // not the last spec item? const s2 = spec[i+1] total *= s2.transform(s2.max) } } return -total } module.exports = { downloadStatus, waitForDownload, videoExists, tryDeleteFromCache, createFfmpegOgg, createThumbStream, // Stolen from yt2009 saveMp4Android, fetchVideoData: async function(id, req) { // Check if video data already in cache let videoData = cacheManager.read("video")[id] if (videoData) return videoData else { // Get video data and set to cache // Uses Cloudtube's api usage stuff const user = getUser(req) const settings = user.getSettingsOrDefaults() var instanceOrigin = settings.instance var outURL = `${instanceOrigin}/api/v1/videos/${id}` let videoFuture = request(outURL) videoData = await videoFuture if (!videoData.ok) return videoData.result videoData = await videoData.result if (videoData.error) return videoData videoData = await module.exports.formatVideoData(videoData, settings) return videoData } }, formatVideoData: async function(videoData, settings) { videoData.formats = await module.exports.sortFormats(videoData, ([480, 720, 1080])[settings.quality]) // process length text and view count for (const rec of videoData.recommendedVideos) converters.normaliseVideoInfo(rec) // normalise view count if (!videoData.second__viewCountText && videoData.viewCount) videoData.second__viewCountText = converters.viewCountToText(videoData.viewCount) // normalize like count if (!videoData.second__likeCountText && videoData.likeCount) videoData.second__likeCountText = converters.likeCountToText(videoData.likeCount) // rewrite description videoData.descriptionHtml = converters.rewriteVideoDescription(videoData.descriptionHtml, videoData.videoId) // rewrite captions urls so they are served on the same domain via the /proxy route for (const caption of videoData.captions) caption.url = `/getCaption?${new URLSearchParams({ url: caption.url })}` // Apply SponsorBlock data if (settings.sponsorblock > 0) { const sbData = await dearrow.getSB(videoData.videoId) if (sbData && !sbData.error) videoData.sbData = sbData.data } // Apply DeArrow data if (settings.dearrow > 0) { const dearrowData = await dearrow.getDeArrow(videoData.videoId) if (dearrowData && !dearrowData.error) videoData.dearrowData = dearrowData.data } return videoData }, sortFormats: async function(video, targetQuality) { // if (video.formatStreams == undefined || video.formatStreams.length == 0) // return [] // Add second__ extensions to format objects, required if Invidious was the extractor let formats = (video.formatStreams || []).concat(video.adaptiveFormats) for (const format of formats) { if (!format.second__height && format.resolution) format.second__height = +format.resolution.slice(0, -1) if (!format.second__order) format.second__order = formatOrder(format) if (!format.container) format.container = format.type.split("video/")[1].split(";")[0] format.cloudtube__label = `${format.qualityLabel} ${format.container}` if (!format.clen) { try { const req = await nodeFetch(format.url, { method: "HEAD" }) format.clen = req.headers.get('content-length') } catch(e) { console.error(e) } } if (format.clen) format.eirtube__size = `${converters.bytesToSizeText(format.clen)}` } // Properly build and order format list let standard = video.formatStreams ? video.formatStreams.slice().sort((a, b) => (b.second__height + (b.container == "mp4" ? -1 : 0)) - (a.second__height + (a.container == "mp4" ? -1 : 0))) : [] let adaptive = video.adaptiveFormats.filter(f => f.type.startsWith("video") && f.qualityLabel).sort((a, b) => a.second__order - b.second__order) for (const format of adaptive) { if (!format.cloudtube__label.endsWith("*")) format.cloudtube__label += " *" } // Sort both groups separately standard = qualitySort.sortFormats(standard, targetQuality, 30) adaptive = qualitySort.sortFormats(adaptive, targetQuality, 30) formats = standard.concat(adaptive) return formats; } }