nixos-configs/devices/nos/modules/llm.nix

44 lines
942 B
Nix
Raw Normal View History

{self, ...}: let
tailscaleIP = "100.64.0.4";
in {
imports = [self.nixosModules.wyoming-plus];
# In case tailscale is down
boot.kernel.sysctl."net.ipv4.ip_nonlocal_bind" = 1;
2024-09-25 15:34:56 -04:00
nixpkgs.overlays = [
# We can get rid of this once full CUDA support works
(final: prev: {
ctranslate2 = prev.ctranslate2.override {
withCUDA = true;
withCuDNN = true;
};
})
];
services = {
# Speech-to-Text
wyoming.faster-whisper.servers."en" = {
enable = true;
uri = "tcp://${tailscaleIP}:10300";
# see https://github.com/rhasspy/wyoming-faster-whisper/releases/tag/v2.0.0
model = "medium";
language = "en";
device = "cuda";
};
# Text-to-Intent
ollama = {
enable = true;
acceleration = "cuda";
host = tailscaleIP;
port = 11434;
loadModels = ["mistral-nemo"];
2024-09-06 10:13:47 -04:00
environmentVariables.OLLAMA_DEBUG = "1";
};
};
}