diff --git a/devices/nos/modules/subtitles/cleanup.nix b/devices/nos/modules/subtitles/cleanup.nix index 470b408..5b12ddb 100644 --- a/devices/nos/modules/subtitles/cleanup.nix +++ b/devices/nos/modules/subtitles/cleanup.nix @@ -1,10 +1,4 @@ -{ - config, - pkgs, - ... -}: let - inherit (config.vars) mainUser; - +{pkgs, ...}: let scriptSrc = pkgs.fetchFromGitHub { owner = "brianspilner01"; repo = "media-server-scripts"; @@ -17,37 +11,17 @@ files = ["${scriptSrc}/sub-clean.sh"]; executable = true; }; -in { - systemd = { - services.sub-clean = { - serviceConfig = { - Type = "oneshot"; - User = mainUser; - Group = config.users.users.${mainUser}.group; - }; +in + pkgs.writeShellApplication { + name = "sub-clean"; - path = [ - pkgs.findutils + runtimeInputs = with pkgs; [ + findutils + gnugrep + gawk + ]; - (pkgs.writeShellApplication { - name = "sub-clean"; - runtimeInputs = with pkgs; [findutils gnugrep gawk]; - text = '' - exec ${script} "$@" - ''; - }) - ]; - - script = '' - find /data/anime -name '*.srt' -exec sub-clean "{}" \; - find /data/movies -name '*.srt' -exec sub-clean "{}" \; - find /data/tv -name '*.srt' -exec sub-clean "{}" \; - ''; - }; - timers.sub-clean = { - wantedBy = ["timers.target"]; - partOf = ["sub-clean.service"]; - timerConfig.OnCalendar = ["0:00:00"]; - }; - }; -} + text = '' + exec ${script} "$@" + ''; + } diff --git a/devices/nos/modules/subtitles/convert.nix b/devices/nos/modules/subtitles/convert.nix new file mode 100644 index 0000000..efa9768 --- /dev/null +++ b/devices/nos/modules/subtitles/convert.nix @@ -0,0 +1,18 @@ +{pkgs, ...}: +pkgs.writeShellApplication { + name = "convertMkv"; + + runtimeInputs = with pkgs; [ + ffmpeg-full + ]; + + text = '' + extension="$1" + file="$2" + + new_file="''${file%."$extension"}.mkv" + + ffmpeg -i "$file" -c copy "$new_file" && + rm "$file" + ''; +} diff --git a/devices/nos/modules/subtitles/default.nix b/devices/nos/modules/subtitles/default.nix index 3fd3451..5386484 100644 --- a/devices/nos/modules/subtitles/default.nix +++ b/devices/nos/modules/subtitles/default.nix @@ -1,6 +1,54 @@ -{...}: { +{ + config, + pkgs, + ... +}: let + inherit (config.vars) mainUser; + + convertMkv = pkgs.callPackage ./convert.nix {inherit pkgs;}; + exportSubs = pkgs.callPackage ./extract-subs {inherit pkgs;}; + sub-clean = pkgs.callPackage ./cleanup.nix {inherit pkgs;}; +in { imports = [ - ./cleanup.nix ./syncing.nix + # TODO: + # - Improve cleanup + # - Sync with bazarr-bulk + # - figure out bazarr postprocessing with subsync ]; + systemd = { + services.manage-subs = { + serviceConfig = { + Type = "oneshot"; + User = mainUser; + Group = config.users.users.${mainUser}.group; + }; + + path = [ + convertMkv + exportSubs + sub-clean + ]; + + script = '' + # Make sure every video file is a mkv + find /data/{anime,history,movies,tv} -name '*.mp4' -exec convertMkv "mp4" "{}" \; + + # Export subs from mkv files + find /data/{anime,history,movies,tv} -name '*.mkv' -printf "%h\0" | + xargs -0 -I '{}' extract-subs '{}' "eng,fre" + + # Remove ads and stuff in subs + find /data/{anime,history,movies,tv} -name '*.srt' -exec sub-clean "{}" \; + ''; + }; + + /* + timers.sub-clean = { + wantedBy = ["timers.target"]; + partOf = ["manage-subs.service"]; + timerConfig.OnCalendar = ["0:00:00"]; + }; + */ + }; } diff --git a/devices/nos/modules/subtitles/extract-subs/.envrc b/devices/nos/modules/subtitles/extract-subs/.envrc new file mode 100644 index 0000000..e74546b --- /dev/null +++ b/devices/nos/modules/subtitles/extract-subs/.envrc @@ -0,0 +1 @@ +use flake $FLAKE#subtitles-dev diff --git a/devices/nos/modules/subtitles/node-syncsub/.eslintrc.json b/devices/nos/modules/subtitles/extract-subs/.eslintrc.json similarity index 100% rename from devices/nos/modules/subtitles/node-syncsub/.eslintrc.json rename to devices/nos/modules/subtitles/extract-subs/.eslintrc.json diff --git a/devices/nos/modules/subtitles/node-syncsub/default.nix b/devices/nos/modules/subtitles/extract-subs/default.nix similarity index 50% rename from devices/nos/modules/subtitles/node-syncsub/default.nix rename to devices/nos/modules/subtitles/extract-subs/default.nix index 2c84e43..685ce90 100644 --- a/devices/nos/modules/subtitles/node-syncsub/default.nix +++ b/devices/nos/modules/subtitles/extract-subs/default.nix @@ -1,21 +1,20 @@ { buildNpmPackage, - ffmpeg, + ffmpeg-full, nodejs_20, - subsync, - symlinkJoin, typescript, + writeShellApplication, ... }: let - nodeSubSync = buildNpmPackage { - name = "node-syncsub"; + pname = "extract-subs"; + + extract-subs = buildNpmPackage { + name = "${pname}-npm"; src = ./.; npmDepsHash = "sha256-O00VQPCUX6T+rtK3VcAibBipXFwNs4AFA3251qycPBQ="; nativeBuildInputs = [ nodejs_20 - ffmpeg - subsync typescript ]; @@ -27,21 +26,26 @@ mkdir -p $out/bin mv node_modules package.json $out - echo '#!/usr/bin/env node' > $out/bin/node-syncsub - cat ./build/main.js >> $out/bin/node-syncsub + echo '#!/usr/bin/env node' > $out/bin/${pname} + cat ./build/main.js >> $out/bin/${pname} rm ./build/main.js - chmod +x $out/bin/node-syncsub + chmod +x $out/bin/${pname} mv ./build/**.js $out/bin ''; }; in - symlinkJoin { - name = "node-syncsub"; - meta.mainProgram = "node-syncsub"; - paths = [ - ffmpeg - subsync - nodeSubSync + writeShellApplication { + name = pname; + + runtimeInputs = [ + ffmpeg-full + extract-subs ]; + + text = '' + exec ${pname} "$@" + ''; + + meta.mainProgram = pname; } diff --git a/devices/nos/modules/subtitles/node-syncsub/lang-codes.ts b/devices/nos/modules/subtitles/extract-subs/lang-codes.ts similarity index 100% rename from devices/nos/modules/subtitles/node-syncsub/lang-codes.ts rename to devices/nos/modules/subtitles/extract-subs/lang-codes.ts diff --git a/devices/nos/modules/subtitles/extract-subs/main.ts b/devices/nos/modules/subtitles/extract-subs/main.ts new file mode 100755 index 0000000..f50c4fc --- /dev/null +++ b/devices/nos/modules/subtitles/extract-subs/main.ts @@ -0,0 +1,113 @@ +import Ffmpeg from 'fluent-ffmpeg'; +import { spawnSync as spawn } from 'child_process'; + +import { ISO6393To1 } from './lang-codes'; + +const SPAWN_OPTS = { + shell: true, + stdio: [process.stdin, process.stdout, process.stderr], +}; + +/** + * These are the cli arguments + * + * @param videoPath the directory in which we want to sync the subtitles + * @param languages a comma-separated list of languages (3 letters) to sync the subtitles + */ +const video = process.argv[2]; +const languages = process.argv[3]?.split(','); + +const escapePath = (p: string): string => p.replaceAll("'", "'\\''"); + +// Check if there are 2 params +if (video && languages) { + main(escapePath(video)); +} +else { + console.error('Error: no argument passed'); + process.exit(1); +} + + +function getSubPath(baseName: string, sub: Ffmpeg.FfprobeStream): string { + const language = ISO6393To1.get(sub.tags.language); + + const forced = sub.disposition?.forced === 0 ? + '' : + '.forced'; + + const hearingImpaired = sub.disposition?.hearing_impaired === 0 ? + '' : + '.sdh'; + + return `${baseName}${forced}.${language}${hearingImpaired}.srt`; +} + +function main(videoPath: string) { + const subIndexes: number[] = []; + const baseName = videoPath.split('/').at(-1)!.replace(/\.[^.]*$/, ''); + + // ffprobe the video file to see available sub tracks + Ffmpeg.ffprobe(videoPath, (_e, data) => { + if (!data?.streams) { + console.error('Couldn\'t find streams in video file'); + + return; + } + + languages.forEach((lang) => { + let subs = data.streams.filter((s) => { + return s['tags'] && + s['tags']['language'] && + s['tags']['language'] === lang && + s.codec_type === 'subtitle'; + }); + + const pgs = subs.filter((s) => s.codec_name === 'hdmv_pgs_subtitle'); + + // If we only have PGS subs, warn user + if (pgs.length === subs.length) { + console.warn(`No SRT subtitle tracks were found for ${lang}`); + } + // Remove PGS streams from subs + subs = subs.filter((s) => s.codec_name !== 'hdmv_pgs_subtitle'); + + if (subs.length === 0) { + console.warn(`No subtitle tracks were found for ${lang}`); + + return; + } + + subs.forEach((sub) => { + const subFile = getSubPath(baseName, sub); + + // Extract subtitle + spawn('ffmpeg', [ + '-i', `'${videoPath}'`, + '-map', `"0:${sub.index}"`, `'${subFile}'`, + ], SPAWN_OPTS); + + subIndexes.push(sub.index); + }); + }); + + // Delete subtitles from video + spawn('mv', [ + `'${videoPath}'`, + `'${videoPath}.bak'`, + ], SPAWN_OPTS); + + spawn('ffmpeg', [ + '-i', `'${videoPath}.bak'`, + '-map', '0', + ...subIndexes.map((i) => [ + '-map', `-0:${i}`, + ]).flat(), + '-c', 'copy', `'${videoPath}'`, + ], SPAWN_OPTS); + + spawn('rm', [ + `'${videoPath}.bak'`, + ], SPAWN_OPTS); + }); +} diff --git a/devices/nos/modules/subtitles/node-syncsub/package-lock.json b/devices/nos/modules/subtitles/extract-subs/package-lock.json similarity index 100% rename from devices/nos/modules/subtitles/node-syncsub/package-lock.json rename to devices/nos/modules/subtitles/extract-subs/package-lock.json diff --git a/devices/nos/modules/subtitles/node-syncsub/package.json b/devices/nos/modules/subtitles/extract-subs/package.json similarity index 100% rename from devices/nos/modules/subtitles/node-syncsub/package.json rename to devices/nos/modules/subtitles/extract-subs/package.json diff --git a/devices/nos/modules/subtitles/node-syncsub/tsconfig.json b/devices/nos/modules/subtitles/extract-subs/tsconfig.json similarity index 79% rename from devices/nos/modules/subtitles/node-syncsub/tsconfig.json rename to devices/nos/modules/subtitles/extract-subs/tsconfig.json index b18570e..41c8a16 100644 --- a/devices/nos/modules/subtitles/node-syncsub/tsconfig.json +++ b/devices/nos/modules/subtitles/extract-subs/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES2022", + "target": "ESNEXT", "module": "commonjs", "lib": [ "ES2022" @@ -9,8 +9,8 @@ "strict": true, "moduleResolution": "node", "baseUrl": ".", - "typeRoots": [ - "./node_modules/@types", + "types": [ + "@types/fluent-ffmpeg" ], "skipLibCheck": true, "forceConsistentCasingInFileNames": true, diff --git a/devices/nos/modules/subtitles/node-syncsub/.envrc b/devices/nos/modules/subtitles/node-syncsub/.envrc deleted file mode 100644 index 2377eaf..0000000 --- a/devices/nos/modules/subtitles/node-syncsub/.envrc +++ /dev/null @@ -1 +0,0 @@ -use flake $FLAKE#node-dev diff --git a/devices/nos/modules/subtitles/node-syncsub/main.ts b/devices/nos/modules/subtitles/node-syncsub/main.ts deleted file mode 100755 index 6461695..0000000 --- a/devices/nos/modules/subtitles/node-syncsub/main.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { - mkdir, - readdir as readDir, - rename as mv, -} from 'fs/promises'; - -import { ffprobe as ffProbe } from 'fluent-ffmpeg'; -import { spawnSync as spawn } from 'child_process'; - -import { ISO6391To3, ISO6393To1 } from './lang-codes'; - -const SPAWN_OPTS = { - shell: true, - stdio: [process.stdin, process.stdout, process.stderr], -}; - -/** - * These are the cli arguments - * - * @param directory the directory in which we want to sync the subtitles - * @param languages a comma-separated list of languages (3 letters) to sync the subtitles - */ -const DIR = process.argv[2]; -const LANGS = process.argv[3]?.split(','); -let langs: string[]; - - -// Check if there are 2 params -if (DIR && LANGS) { - main(); -} -else { - console.error('Error: no argument passed'); - process.exit(1); -} - -const escapePath = (p: string): string => p.replaceAll("'", "'\\''"); - -function getVideoPath(files: string[]): string[] { - const fileName = DIR.split('/').at(-1) ?? ''; - - const videoFiles = files.filter((f) => - f.includes(fileName) && f.endsWith('mkv')); - - if (videoFiles.length === 0) { - console.warn('No video files were found'); - process.exit(0); - } - - return videoFiles.map((file) => `${DIR}/${file}`); -} - -async function backupSubs(files: string[], base: string) { - // Check if backup folder already exists and create it if not - if (!files.some((f) => f.endsWith('.srt.bak'))) { - await mkdir(`${DIR}/.srt.bak`); - } - else { - // TODO: compare with subs outside of backup dir - const backups = await readDir(`${DIR}/.srt.bak`); - - // Remove synced subtitles from the list to sync - // langs - backups - langs = langs - .filter((n) => !backups - .filter((l) => l.includes(base)) - .some((s) => { - const l2 = s.split('.').at(-2) ?? ''; - const l3 = ISO6391To3.get(l2); - - return n === l3; - })); - } - - if (langs.length === 0) { - console.warn(`Subtitles have already been synced for ${base}`); - } -} - -async function runSubSync( - cmd: string[], - input: string, - output: string, -) { - spawn('subsync', cmd, SPAWN_OPTS); - - if (!(await readDir(DIR)).includes(output)) { - await mv(input, output); - console.log('Subtitle was moved back'); - } - - spawn('chmod', ['-R', '775', `'${escapePath(DIR)}'`], SPAWN_OPTS); -} - -async function main() { - const files = await readDir(DIR); - - const VIDEO_FILES = getVideoPath(files); - - VIDEO_FILES.forEach((VIDEO) => { - langs = LANGS; - const BASE_NAME = VIDEO.split('/').at(-1)!.replace(/\.[^.]*$/, ''); - - backupSubs(files, BASE_NAME); - - // ffprobe the video file to see available audio tracks - ffProbe(VIDEO, (_e, data) => { - if (!data?.streams) { - console.error('Couldn\'t find streams in video file'); - } - else { - const AVAIL_LANGS = data.streams - .filter((s) => s.codec_type === 'audio') - .map((s) => s['tags'] && s['tags']['language']); - - // Sync subtitles one by one - langs.forEach(async(lang) => { - const FILE_NAME = `${BASE_NAME}.${ISO6393To1.get(lang)}.srt`; - const IN_FILE = `${DIR}/.srt.bak/${FILE_NAME}`; - const OUT_FILE = `${DIR}/${FILE_NAME}`; - - const cmd = [ - '--cli sync', - `--sub-lang ${lang}`, - - `--ref-stream-by-lang ${AVAIL_LANGS.includes(lang) ? - lang : - AVAIL_LANGS[0]}`, - '--ref-stream-by-type "audio"', - - `--sub '${escapePath(IN_FILE)}'`, - `--out '${escapePath(OUT_FILE)}'`, - `--ref '${escapePath(VIDEO)}'`, - ]; - - if (files.includes(FILE_NAME)) { - await mv(OUT_FILE, IN_FILE); - - runSubSync(cmd, IN_FILE, OUT_FILE); - } - else { - let subs = data.streams.filter((s) => { - return s['tags'] && - s['tags']['language'] && - s['tags']['language'] === lang && - s.codec_type === 'subtitle'; - }); - - const pgs = subs.filter((s) => s.codec_name === 'hdmv_pgs_subtitle'); - - // If we only have PGS subs, warn user - if (pgs.length === subs.length) { - console.warn(`No SRT subtitle tracks were found for ${lang}`); - } - // Remove PGS streams from subs - subs = subs.filter((s) => s.codec_name !== 'hdmv_pgs_subtitle'); - - // Prefer normal subs - if (subs.length !== 1) { - subs = subs.filter((s) => s.disposition?.forced === 0); - } - - if (subs.length === 0) { - console.warn(`No subtitle tracks were found for ${lang}`); - } - else { - // Extract subtitle - spawn('ffmpeg', [ - '-i', `'${escapePath(VIDEO)}'`, - '-map', `"0:${subs[0].index}"`, `'${escapePath(IN_FILE)}'`, - ], SPAWN_OPTS); - - // Delete subtitle from video - spawn('mv', [ - `'${escapePath(VIDEO)}'`, - `'${escapePath(VIDEO)}.bak'`, - ], SPAWN_OPTS); - - spawn('ffmpeg', [ - '-i', `'${escapePath(VIDEO)}.bak'`, - '-map', '0', - '-map', `-0:${subs[0].index}`, - '-c', 'copy', `'${escapePath(VIDEO)}'`, - ], SPAWN_OPTS); - - spawn('rm', [`'${escapePath(VIDEO)}.bak'`], SPAWN_OPTS); - - // Sync extracted subtitle - runSubSync(cmd, IN_FILE, OUT_FILE); - } - } - }); - } - }); - }); -} diff --git a/devices/nos/modules/subtitles/syncing.nix b/devices/nos/modules/subtitles/syncing.nix index 1df7ca9..42d7b38 100644 --- a/devices/nos/modules/subtitles/syncing.nix +++ b/devices/nos/modules/subtitles/syncing.nix @@ -3,17 +3,9 @@ config, lib, pkgs, - subsync, ... }: let - inherit (config.vars) mainUser; - - subsyncPkg = subsync.packages.${pkgs.system}.default; bbPkg = bazarr-bulk.packages.${pkgs.system}.default; - - node-syncsub = pkgs.callPackage ./node-syncsub { - subsync = subsyncPkg; - }; in { environment.systemPackages = [ (pkgs.writeShellApplication { @@ -23,30 +15,4 @@ in { ''; }) ]; - - systemd = { - services.subsync-job = { - serviceConfig = { - Type = "oneshot"; - User = mainUser; - Group = config.users.users.${mainUser}.group; - }; - - path = [ - pkgs.findutils - node-syncsub - ]; - - script = '' - find /data/movies -name '*.mkv' -printf "%h\0" | xargs -0 -I '{}' node-syncsub '{}' "eng,fre" - # find /data/anime -name '*.mkv' -printf "%h\0" | xargs -0 -I '{}' node-syncsub '{}' "eng,fre" - # find /data/tv -name '*.mkv' -printf "%h\0" | xargs -0 -I '{}' node-syncsub '{}' "eng,fre" - ''; - }; - #timers.subsync-job = { - # wantedBy = ["timers.target"]; - # partOf = ["subsync-job.service"]; - # timerConfig.OnCalendar = ["0:00:00"]; - #}; - }; } diff --git a/flake.nix b/flake.nix index f5d5fc1..e1b6570 100644 --- a/flake.nix +++ b/flake.nix @@ -106,11 +106,11 @@ ]; }; - node-dev = pkgs.mkShell { + subtitles-dev = pkgs.mkShell { packages = with pkgs; [ nodejs_latest - ffmpeg + ffmpeg-full typescript ] ++ (with nodePackages; [