feat(subs): separate mgmt of subs in scripts

devices/nos/modules/subtitles/cleanup.nix

@@ -1,10 +1,4 @@
-{
+{pkgs, ...}: let
-  config,
-  pkgs,
-  ...
-}: let
-  inherit (config.vars) mainUser;
-
   scriptSrc = pkgs.fetchFromGitHub {
     owner = "brianspilner01";
     repo = "media-server-scripts";
@@ -17,37 +11,17 @@
     files = ["${scriptSrc}/sub-clean.sh"];
     executable = true;
   };
-in {
+in
-  systemd = {
+  pkgs.writeShellApplication {
-    services.sub-clean = {
-      serviceConfig = {
-        Type = "oneshot";
-        User = mainUser;
-        Group = config.users.users.${mainUser}.group;
-      };
-
-      path = [
-        pkgs.findutils
-
-        (pkgs.writeShellApplication {
     name = "sub-clean";
-          runtimeInputs = with pkgs; [findutils gnugrep gawk];
+
+    runtimeInputs = with pkgs; [
+      findutils
+      gnugrep
+      gawk
+    ];
+
     text = ''
       exec ${script} "$@"
     '';
-        })
+  }
-      ];
-
-      script = ''
-        find /data/anime  -name '*.srt' -exec sub-clean "{}" \;
-        find /data/movies -name '*.srt' -exec sub-clean "{}" \;
-        find /data/tv     -name '*.srt' -exec sub-clean "{}" \;
-      '';
-    };
-    timers.sub-clean = {
-      wantedBy = ["timers.target"];
-      partOf = ["sub-clean.service"];
-      timerConfig.OnCalendar = ["0:00:00"];
-    };
-  };
-}

devices/nos/modules/subtitles/convert.nix

@@ -0,0 +1,18 @@
+{pkgs, ...}:
+pkgs.writeShellApplication {
+  name = "convertMkv";
+
+  runtimeInputs = with pkgs; [
+    ffmpeg-full
+  ];
+
+  text = ''
+    extension="$1"
+    file="$2"
+
+    new_file="''${file%."$extension"}.mkv"
+
+    ffmpeg -i "$file" -c copy "$new_file" &&
+    rm "$file"
+  '';
+}

devices/nos/modules/subtitles/default.nix

@@ -1,6 +1,54 @@
-{...}: {
+{
+  config,
+  pkgs,
+  ...
+}: let
+  inherit (config.vars) mainUser;
+
+  convertMkv = pkgs.callPackage ./convert.nix {inherit pkgs;};
+  exportSubs = pkgs.callPackage ./extract-subs {inherit pkgs;};
+  sub-clean = pkgs.callPackage ./cleanup.nix {inherit pkgs;};
+in {
   imports = [
-    ./cleanup.nix
     ./syncing.nix
+    # TODO:
+    # - Improve cleanup
+    # - Sync with bazarr-bulk
+    # - figure out bazarr postprocessing with subsync
   ];
+  systemd = {
+    services.manage-subs = {
+      serviceConfig = {
+        Type = "oneshot";
+        User = mainUser;
+        Group = config.users.users.${mainUser}.group;
+      };
+
+      path = [
+        convertMkv
+        exportSubs
+        sub-clean
+      ];
+
+      script = ''
+        # Make sure every video file is a mkv
+        find /data/{anime,history,movies,tv} -name '*.mp4' -exec convertMkv "mp4" "{}" \;
+
+        # Export subs from mkv files
+        find /data/{anime,history,movies,tv} -name '*.mkv' -printf "%h\0" |
+        xargs -0 -I '{}' extract-subs '{}' "eng,fre"
+
+        # Remove ads and stuff in subs
+        find /data/{anime,history,movies,tv} -name '*.srt' -exec sub-clean "{}" \;
+      '';
+    };
+
+    /*
+    timers.sub-clean = {
+      wantedBy = ["timers.target"];
+      partOf = ["manage-subs.service"];
+      timerConfig.OnCalendar = ["0:00:00"];
+    };
+    */
+  };
 }

devices/nos/modules/subtitles/extract-subs/.envrc

@@ -0,0 +1 @@
+use flake $FLAKE#subtitles-dev

devices/nos/modules/subtitles/extract-subs/default.nix

@@ -1,21 +1,20 @@
 {
   buildNpmPackage,
-  ffmpeg,
+  ffmpeg-full,
   nodejs_20,
-  subsync,
-  symlinkJoin,
   typescript,
+  writeShellApplication,
   ...
 }: let
-  nodeSubSync = buildNpmPackage {
+  pname = "extract-subs";
-    name = "node-syncsub";
+
+  extract-subs = buildNpmPackage {
+    name = "${pname}-npm";
     src = ./.;
     npmDepsHash = "sha256-O00VQPCUX6T+rtK3VcAibBipXFwNs4AFA3251qycPBQ=";
 
     nativeBuildInputs = [
       nodejs_20
-      ffmpeg
-      subsync
       typescript
     ];
 
@@ -27,21 +26,26 @@
       mkdir -p $out/bin
       mv node_modules package.json $out
 
-      echo '#!/usr/bin/env node' > $out/bin/node-syncsub
+      echo '#!/usr/bin/env node' > $out/bin/${pname}
-      cat ./build/main.js >> $out/bin/node-syncsub
+      cat ./build/main.js >> $out/bin/${pname}
       rm ./build/main.js
-      chmod +x $out/bin/node-syncsub
+      chmod +x $out/bin/${pname}
 
       mv ./build/**.js $out/bin
     '';
   };
 in
-  symlinkJoin {
+  writeShellApplication {
-    name = "node-syncsub";
+    name = pname;
-    meta.mainProgram = "node-syncsub";
+
-    paths = [
+    runtimeInputs = [
-      ffmpeg
+      ffmpeg-full
-      subsync
+      extract-subs
-      nodeSubSync
     ];
+
+    text = ''
+      exec ${pname} "$@"
+    '';
+
+    meta.mainProgram = pname;
   }

devices/nos/modules/subtitles/extract-subs/main.ts

@@ -0,0 +1,113 @@
+import Ffmpeg from 'fluent-ffmpeg';
+import { spawnSync as spawn } from 'child_process';
+
+import { ISO6393To1 } from './lang-codes';
+
+const SPAWN_OPTS = {
+    shell: true,
+    stdio: [process.stdin, process.stdout, process.stderr],
+};
+
+/**
+ * These are the cli arguments
+ *
+ * @param videoPath the directory in which we want to sync the subtitles
+ * @param languages a comma-separated list of languages (3 letters) to sync the subtitles
+ */
+const video = process.argv[2];
+const languages = process.argv[3]?.split(',');
+
+const escapePath = (p: string): string => p.replaceAll("'", "'\\''");
+
+// Check if there are 2 params
+if (video && languages) {
+    main(escapePath(video));
+}
+else {
+    console.error('Error: no argument passed');
+    process.exit(1);
+}
+
+
+function getSubPath(baseName: string, sub: Ffmpeg.FfprobeStream): string {
+    const language = ISO6393To1.get(sub.tags.language);
+
+    const forced = sub.disposition?.forced === 0 ?
+        '' :
+        '.forced';
+
+    const hearingImpaired = sub.disposition?.hearing_impaired === 0 ?
+        '' :
+        '.sdh';
+
+    return `${baseName}${forced}.${language}${hearingImpaired}.srt`;
+}
+
+function main(videoPath: string) {
+    const subIndexes: number[] = [];
+    const baseName = videoPath.split('/').at(-1)!.replace(/\.[^.]*$/, '');
+
+    // ffprobe the video file to see available sub tracks
+    Ffmpeg.ffprobe(videoPath, (_e, data) => {
+        if (!data?.streams) {
+            console.error('Couldn\'t find streams in video file');
+
+            return;
+        }
+
+        languages.forEach((lang) => {
+            let subs = data.streams.filter((s) => {
+                return s['tags'] &&
+                    s['tags']['language'] &&
+                    s['tags']['language'] === lang &&
+                    s.codec_type === 'subtitle';
+            });
+
+            const pgs = subs.filter((s) => s.codec_name === 'hdmv_pgs_subtitle');
+
+            // If we only have PGS subs, warn user
+            if (pgs.length === subs.length) {
+                console.warn(`No SRT subtitle tracks were found for ${lang}`);
+            }
+            // Remove PGS streams from subs
+            subs = subs.filter((s) => s.codec_name !== 'hdmv_pgs_subtitle');
+
+            if (subs.length === 0) {
+                console.warn(`No subtitle tracks were found for ${lang}`);
+
+                return;
+            }
+
+            subs.forEach((sub) => {
+                const subFile = getSubPath(baseName, sub);
+
+                // Extract subtitle
+                spawn('ffmpeg', [
+                    '-i', `'${videoPath}'`,
+                    '-map', `"0:${sub.index}"`, `'${subFile}'`,
+                ], SPAWN_OPTS);
+
+                subIndexes.push(sub.index);
+            });
+        });
+
+        // Delete subtitles from video
+        spawn('mv', [
+            `'${videoPath}'`,
+            `'${videoPath}.bak'`,
+        ], SPAWN_OPTS);
+
+        spawn('ffmpeg', [
+            '-i', `'${videoPath}.bak'`,
+            '-map', '0',
+            ...subIndexes.map((i) => [
+                '-map', `-0:${i}`,
+            ]).flat(),
+            '-c', 'copy', `'${videoPath}'`,
+        ], SPAWN_OPTS);
+
+        spawn('rm', [
+            `'${videoPath}.bak'`,
+        ], SPAWN_OPTS);
+    });
+}

devices/nos/modules/subtitles/extract-subs/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "ES2022",
+    "target": "ESNEXT",
     "module": "commonjs",
     "lib": [
       "ES2022"
@@ -9,8 +9,8 @@
     "strict": true,
     "moduleResolution": "node",
     "baseUrl": ".",
-    "typeRoots": [
+    "types": [
-      "./node_modules/@types",
+      "@types/fluent-ffmpeg"
     ],
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,

devices/nos/modules/subtitles/node-syncsub/.envrc

@@ -1 +0,0 @@
-use flake $FLAKE#node-dev

devices/nos/modules/subtitles/node-syncsub/main.ts

@@ -1,196 +0,0 @@
-import {
-    mkdir,
-    readdir as readDir,
-    rename as mv,
-} from 'fs/promises';
-
-import { ffprobe as ffProbe } from 'fluent-ffmpeg';
-import { spawnSync as spawn } from 'child_process';
-
-import { ISO6391To3, ISO6393To1 } from './lang-codes';
-
-const SPAWN_OPTS = {
-    shell: true,
-    stdio: [process.stdin, process.stdout, process.stderr],
-};
-
-/**
- * These are the cli arguments
- *
- * @param directory the directory in which we want to sync the subtitles
- * @param languages a comma-separated list of languages (3 letters) to sync the subtitles
- */
-const DIR = process.argv[2];
-const LANGS = process.argv[3]?.split(',');
-let langs: string[];
-
-
-// Check if there are 2 params
-if (DIR && LANGS) {
-    main();
-}
-else {
-    console.error('Error: no argument passed');
-    process.exit(1);
-}
-
-const escapePath = (p: string): string => p.replaceAll("'", "'\\''");
-
-function getVideoPath(files: string[]): string[] {
-    const fileName = DIR.split('/').at(-1) ?? '';
-
-    const videoFiles = files.filter((f) =>
-        f.includes(fileName) && f.endsWith('mkv'));
-
-    if (videoFiles.length === 0) {
-        console.warn('No video files were found');
-        process.exit(0);
-    }
-
-    return videoFiles.map((file) => `${DIR}/${file}`);
-}
-
-async function backupSubs(files: string[], base: string) {
-    // Check if backup folder already exists and create it if not
-    if (!files.some((f) => f.endsWith('.srt.bak'))) {
-        await mkdir(`${DIR}/.srt.bak`);
-    }
-    else {
-        // TODO: compare with subs outside of backup dir
-        const backups = await readDir(`${DIR}/.srt.bak`);
-
-        // Remove synced subtitles from the list to sync
-        // langs - backups
-        langs = langs
-            .filter((n) => !backups
-                .filter((l) => l.includes(base))
-                .some((s) => {
-                    const l2 = s.split('.').at(-2) ?? '';
-                    const l3 = ISO6391To3.get(l2);
-
-                    return n === l3;
-                }));
-    }
-
-    if (langs.length === 0) {
-        console.warn(`Subtitles have already been synced for ${base}`);
-    }
-}
-
-async function runSubSync(
-    cmd: string[],
-    input: string,
-    output: string,
-) {
-    spawn('subsync', cmd, SPAWN_OPTS);
-
-    if (!(await readDir(DIR)).includes(output)) {
-        await mv(input, output);
-        console.log('Subtitle was moved back');
-    }
-
-    spawn('chmod', ['-R', '775', `'${escapePath(DIR)}'`], SPAWN_OPTS);
-}
-
-async function main() {
-    const files = await readDir(DIR);
-
-    const VIDEO_FILES = getVideoPath(files);
-
-    VIDEO_FILES.forEach((VIDEO) => {
-        langs = LANGS;
-        const BASE_NAME = VIDEO.split('/').at(-1)!.replace(/\.[^.]*$/, '');
-
-        backupSubs(files, BASE_NAME);
-
-        // ffprobe the video file to see available audio tracks
-        ffProbe(VIDEO, (_e, data) => {
-            if (!data?.streams) {
-                console.error('Couldn\'t find streams in video file');
-            }
-            else {
-                const AVAIL_LANGS = data.streams
-                    .filter((s) => s.codec_type === 'audio')
-                    .map((s) => s['tags'] && s['tags']['language']);
-
-                // Sync subtitles one by one
-                langs.forEach(async(lang) => {
-                    const FILE_NAME = `${BASE_NAME}.${ISO6393To1.get(lang)}.srt`;
-                    const IN_FILE = `${DIR}/.srt.bak/${FILE_NAME}`;
-                    const OUT_FILE = `${DIR}/${FILE_NAME}`;
-
-                    const cmd = [
-                        '--cli sync',
-                        `--sub-lang ${lang}`,
-
-                        `--ref-stream-by-lang ${AVAIL_LANGS.includes(lang) ?
-                            lang :
-                            AVAIL_LANGS[0]}`,
-                        '--ref-stream-by-type "audio"',
-
-                        `--sub '${escapePath(IN_FILE)}'`,
-                        `--out '${escapePath(OUT_FILE)}'`,
-                        `--ref '${escapePath(VIDEO)}'`,
-                    ];
-
-                    if (files.includes(FILE_NAME)) {
-                        await mv(OUT_FILE, IN_FILE);
-
-                        runSubSync(cmd, IN_FILE, OUT_FILE);
-                    }
-                    else {
-                        let subs = data.streams.filter((s) => {
-                            return s['tags'] &&
-                                s['tags']['language'] &&
-                                s['tags']['language'] === lang &&
-                                s.codec_type === 'subtitle';
-                        });
-
-                        const pgs = subs.filter((s) => s.codec_name === 'hdmv_pgs_subtitle');
-
-                        // If we only have PGS subs, warn user
-                        if (pgs.length === subs.length) {
-                            console.warn(`No SRT subtitle tracks were found for ${lang}`);
-                        }
-                        // Remove PGS streams from subs
-                        subs = subs.filter((s) => s.codec_name !== 'hdmv_pgs_subtitle');
-
-                        // Prefer normal subs
-                        if (subs.length !== 1) {
-                            subs = subs.filter((s) => s.disposition?.forced === 0);
-                        }
-
-                        if (subs.length === 0) {
-                            console.warn(`No subtitle tracks were found for ${lang}`);
-                        }
-                        else {
-                            // Extract subtitle
-                            spawn('ffmpeg', [
-                                '-i', `'${escapePath(VIDEO)}'`,
-                                '-map', `"0:${subs[0].index}"`, `'${escapePath(IN_FILE)}'`,
-                            ], SPAWN_OPTS);
-
-                            // Delete subtitle from video
-                            spawn('mv', [
-                                `'${escapePath(VIDEO)}'`,
-                                `'${escapePath(VIDEO)}.bak'`,
-                            ], SPAWN_OPTS);
-
-                            spawn('ffmpeg', [
-                                '-i', `'${escapePath(VIDEO)}.bak'`,
-                                '-map', '0',
-                                '-map', `-0:${subs[0].index}`,
-                                '-c', 'copy', `'${escapePath(VIDEO)}'`,
-                            ], SPAWN_OPTS);
-
-                            spawn('rm', [`'${escapePath(VIDEO)}.bak'`], SPAWN_OPTS);
-
-                            // Sync extracted subtitle
-                            runSubSync(cmd, IN_FILE, OUT_FILE);
-                        }
-                    }
-                });
-            }
-        });
-    });
-}

devices/nos/modules/subtitles/syncing.nix

@@ -3,17 +3,9 @@
   config,
   lib,
   pkgs,
-  subsync,
   ...
 }: let
-  inherit (config.vars) mainUser;
-
-  subsyncPkg = subsync.packages.${pkgs.system}.default;
   bbPkg = bazarr-bulk.packages.${pkgs.system}.default;
-
-  node-syncsub = pkgs.callPackage ./node-syncsub {
-    subsync = subsyncPkg;
-  };
 in {
   environment.systemPackages = [
     (pkgs.writeShellApplication {
@@ -23,30 +15,4 @@ in {
       '';
     })
   ];
-
-  systemd = {
-    services.subsync-job = {
-      serviceConfig = {
-        Type = "oneshot";
-        User = mainUser;
-        Group = config.users.users.${mainUser}.group;
-      };
-
-      path = [
-        pkgs.findutils
-        node-syncsub
-      ];
-
-      script = ''
-        find /data/movies -name '*.mkv' -printf "%h\0" | xargs -0 -I '{}' node-syncsub '{}' "eng,fre"
-        # find /data/anime -name '*.mkv' -printf "%h\0" | xargs -0 -I '{}' node-syncsub '{}' "eng,fre"
-        # find /data/tv -name '*.mkv' -printf "%h\0" | xargs -0 -I '{}' node-syncsub '{}' "eng,fre"
-      '';
-    };
-    #timers.subsync-job = {
-    #  wantedBy = ["timers.target"];
-    #  partOf = ["subsync-job.service"];
-    #  timerConfig.OnCalendar = ["0:00:00"];
-    #};
-  };
 }