diff --git a/flake.lock b/flake.lock index cff556c..62f241f 100644 --- a/flake.lock +++ b/flake.lock @@ -125,7 +125,23 @@ "inputs": { "flake-parts": "flake-parts", "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_2", + "vpnconfinement": "vpnconfinement" + } + }, + "vpnconfinement": { + "locked": { + "lastModified": 1778182451, + "narHash": "sha256-Bz3n2THDGf90Z9gMqhH/J492prYH8B6RFRlxv/fPBwc=", + "owner": "Maroka-chan", + "repo": "VPN-Confinement", + "rev": "cf5bfc4c3559f2e783698b1aa23c165072039a7d", + "type": "github" + }, + "original": { + "owner": "Maroka-chan", + "repo": "VPN-Confinement", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 8c3e521..24defb5 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,7 @@ nixpkgs.url = "github:nixos/nixpkgs"; flake-parts.url = "github:hercules-ci/flake-parts"; git-hooks.url = "github:cachix/git-hooks.nix"; + vpnconfinement.url = "github:Maroka-chan/VPN-Confinement"; }; outputs = @@ -18,6 +19,18 @@ "x86_64-linux" ]; + flake = { + nixosModules = { + agregators = { + imports = [ + ./nix/default.nix + inputs.vpnconfinement.nixosModules.default + ]; + }; + default = self.nixosModules.agregators; + }; + }; + perSystem = { system, @@ -49,6 +62,9 @@ checks = { inherit pre-commit-check; + simple-test = pkgs.callPackage ./tests/simple-test.nix { + nixosModules = self.nixosModules; + }; }; devShells.default = pkgs.mkShell { diff --git a/nix/default.nix b/nix/default.nix new file mode 100644 index 0000000..44817d4 --- /dev/null +++ b/nix/default.nix @@ -0,0 +1,68 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; +in +{ + imports = [ + ./postgres.nix + ./vpn.nix + ./qbittorrent.nix + ./jackett.nix + ./metadata-agregator.nix + ./musicfs.nix + ./music-agregator.nix + ]; + + options.services.agregators = { + enable = lib.mkEnableOption "agregators music automation pipeline"; + + mediaDir = lib.mkOption { + type = lib.types.path; + default = "/data/music"; + description = '' + Root directory for music files. + qBittorrent downloads here, musicfs reads from here. + ''; + }; + + stateDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/agregators"; + description = "Root state directory. Per-service subdirectories are created automatically."; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "agregators"; + description = "Shared system user for all agregators services."; + }; + + group = lib.mkOption { + type = lib.types.str; + default = "agregators"; + description = "Shared system group for all agregators services."; + }; + }; + + config = lib.mkIf cfg.enable { + users.users.${cfg.user} = lib.mkIf (cfg.user == "agregators") { + isSystemUser = true; + group = cfg.group; + home = cfg.stateDir; + }; + + users.groups.${cfg.group} = lib.mkIf (cfg.group == "agregators") { }; + + systemd.tmpfiles.rules = [ + "d '${cfg.mediaDir}' 0775 ${cfg.user} ${cfg.group} - -" + "d '${cfg.stateDir}' 0750 ${cfg.user} ${cfg.group} - -" + "d '${cfg.stateDir}/postgres' 0770 ${cfg.user} ${cfg.group} - -" + ]; + }; +} diff --git a/nix/jackett.nix b/nix/jackett.nix new file mode 100644 index 0000000..9afb03c --- /dev/null +++ b/nix/jackett.nix @@ -0,0 +1,75 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; + jackettCfg = cfg.jackett; +in +{ + options.services.agregators.jackett = { + enable = lib.mkEnableOption "Jackett torrent indexer aggregator"; + + port = lib.mkOption { + type = lib.types.port; + default = 9117; + description = "Jackett HTTP port."; + }; + + stateDir = lib.mkOption { + type = lib.types.path; + default = "${cfg.stateDir}/jackett"; + defaultText = lib.literalExpression ''"''${cfg.stateDir}/jackett"''; + description = "Jackett data directory."; + }; + + apiKeyFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + File containing the Jackett API key. + If null, music-agregator must be configured with the API key manually. + ''; + }; + + package = lib.mkPackageOption pkgs "jackett" { }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the Jackett port in the firewall."; + }; + }; + + config = lib.mkIf (cfg.enable && jackettCfg.enable) { + systemd.tmpfiles.rules = [ + "d '${jackettCfg.stateDir}' 0750 ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.jackett = { + description = "Jackett torrent indexer aggregator"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + ExecStart = lib.concatStringsSep " " [ + "${lib.getExe jackettCfg.package}" + "--DataFolder=${jackettCfg.stateDir}" + "--Port=${toString jackettCfg.port}" + "--NoUpdates" + ]; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf jackettCfg.openFirewall [ jackettCfg.port ]; + }; +} diff --git a/nix/metadata-agregator.nix b/nix/metadata-agregator.nix new file mode 100644 index 0000000..de38476 --- /dev/null +++ b/nix/metadata-agregator.nix @@ -0,0 +1,142 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; + metaCfg = cfg.metadata-agregator; + pgCfg = cfg.postgres; + db = pgCfg.metadataDatabase; + yaml = pkgs.formats.yaml { }; + + dbPasswordPlaceholder = "@META_DB_PASSWORD@"; + + configFile = yaml.generate "metadata-agregator.yaml" { + server.port = metaCfg.port; + database = { + host = if pgCfg.host == "/run/postgresql" then "localhost" else pgCfg.host; + port = pgCfg.port; + name = db.name; + user = db.user; + password = dbPasswordPlaceholder; + sslmode = "disable"; + }; + logging = { + level = metaCfg.logging.level; + format = metaCfg.logging.format; + }; + metrics = { + enabled = metaCfg.metrics.enable; + port = metaCfg.metrics.port; + }; + }; +in +{ + options.services.agregators.metadata-agregator = { + enable = lib.mkEnableOption "metadata-agregator music metadata gRPC service"; + + package = lib.mkOption { + type = lib.types.package; + description = "The metadata-agregator package."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 50051; + description = "gRPC listen port."; + }; + + logging = { + level = lib.mkOption { + type = lib.types.str; + default = "info"; + description = "Log level."; + }; + + format = lib.mkOption { + type = lib.types.enum [ + "json" + "console" + ]; + default = "json"; + description = "Log output format."; + }; + }; + + metrics = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Enable Prometheus metrics endpoint."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 9091; + description = "Prometheus metrics HTTP port."; + }; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the gRPC port in the firewall."; + }; + }; + + config = lib.mkIf (cfg.enable && metaCfg.enable) { + systemd.services.metadata-agregator = { + description = "metadata-agregator - Music metadata gRPC service"; + after = [ + "network.target" + "agregators-schema-metadata.service" + ]; + requires = [ "agregators-schema-metadata.service" ]; + wantedBy = [ "multi-user.target" ]; + + preStart = '' + cp --no-preserve=mode ${configFile} /run/metadata-agregator/config.yaml + + ${ + if db.passwordFile != null then + '' + ${pkgs.replace-secret}/bin/replace-secret \ + '${dbPasswordPlaceholder}' \ + '${db.passwordFile}' \ + /run/metadata-agregator/config.yaml + '' + else + '' + ${pkgs.gnused}/bin/sed -i "s/${dbPasswordPlaceholder}//" /run/metadata-agregator/config.yaml + '' + } + ''; + + serviceConfig = { + Type = "simple"; + ExecStart = "${lib.getExe metaCfg.package} -config /run/metadata-agregator/config.yaml"; + User = cfg.user; + Group = cfg.group; + RuntimeDirectory = "metadata-agregator"; + RuntimeDirectoryMode = "0750"; + Restart = "on-failure"; + RestartSec = 5; + + ProtectSystem = "strict"; + PrivateTmp = true; + NoNewPrivileges = true; + ProtectHome = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + }; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf metaCfg.openFirewall ( + [ metaCfg.port ] ++ lib.optional metaCfg.metrics.enable metaCfg.metrics.port + ); + }; +} diff --git a/nix/music-agregator.nix b/nix/music-agregator.nix new file mode 100644 index 0000000..42565b1 --- /dev/null +++ b/nix/music-agregator.nix @@ -0,0 +1,275 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; + maCfg = cfg.music-agregator; + pgCfg = cfg.postgres; + qbtCfg = cfg.qbittorrent; + jackettCfg = cfg.jackett; + metaCfg = cfg.metadata-agregator; + mfsCfg = cfg.musicfs; + vpnCfg = cfg.vpn; + yaml = pkgs.formats.yaml { }; + + vpnEnabled = vpnCfg.enable && qbtCfg.vpn.enable or false; + namespaceAddr = "192.168.15.1"; + musicfsEnabled = mfsCfg.enable; + + dbPasswordPlaceholder = "@MUSIC_DB_PASSWORD@"; + qbtPasswordPlaceholder = "@QBT_PASSWORD@"; + + # Auto-wire qBittorrent address based on VPN + qbtAddr = + if maCfg.qbittorrentUrl != null then + maCfg.qbittorrentUrl + else if vpnEnabled then + "http://${namespaceAddr}:${toString qbtCfg.webuiPort}" + else + "http://127.0.0.1:${toString qbtCfg.webuiPort}"; + + dbHost = if pgCfg.host == "/run/postgresql" then "localhost" else pgCfg.host; + + configFile = yaml.generate "music-agregator.yaml" { + app = { + host = maCfg.host; + port = toString maCfg.port; + }; + database = { + url = "postgresql://${pgCfg.musicDatabase.user}:${dbPasswordPlaceholder}@${dbHost}:${toString pgCfg.port}/${pgCfg.musicDatabase.name}?sslmode=disable"; + }; + indexer = { + url = maCfg.jackettUrl; + type = "jackett"; + api_key = "@JACKETT_API_KEY@"; + } + // lib.optionalAttrs (maCfg.indexerCache.enable) { + cache = { + enabled = true; + refresh_interval = maCfg.indexerCache.refreshInterval; + ttl = maCfg.indexerCache.ttl; + }; + }; + torrent = { + client_type = "qbittorrent"; + url = qbtAddr; + username = qbtCfg.username; + password = qbtPasswordPlaceholder; + # No container_name — native paths, no Docker + }; + metadata = { + endpoint = maCfg.metadataEndpoint; + }; + musicfs = { + enabled = musicfsEnabled; + endpoint = maCfg.musicfsEndpoint; + origin_id = lib.optionalString musicfsEnabled mfsCfg.originId; + origin_root = lib.optionalString musicfsEnabled (toString mfsCfg.originPath); + timeout_seconds = maCfg.musicfsTimeoutSeconds; + }; + }; +in +{ + options.services.agregators.music-agregator = { + enable = lib.mkEnableOption "music-agregator orchestrator service"; + + package = lib.mkOption { + type = lib.types.package; + description = "The music-agregator package."; + }; + + host = lib.mkOption { + type = lib.types.str; + default = "0.0.0.0"; + description = "gRPC server bind address."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 3000; + description = "gRPC server port."; + }; + + # --- Auto-wired endpoints (override for external services) --- + + metadataEndpoint = lib.mkOption { + type = lib.types.str; + default = "localhost:${toString metaCfg.port}"; + defaultText = lib.literalExpression ''"localhost:''${toString cfg.metadata-agregator.port}"''; + description = "metadata-agregator gRPC endpoint."; + }; + + musicfsEndpoint = lib.mkOption { + type = lib.types.str; + default = "localhost:${toString mfsCfg.port}"; + defaultText = lib.literalExpression ''"localhost:''${toString cfg.musicfs.port}"''; + description = "musicfs gRPC endpoint."; + }; + + musicfsTimeoutSeconds = lib.mkOption { + type = lib.types.int; + default = 300; + description = "Timeout in seconds for musicfs operations."; + }; + + jackettUrl = lib.mkOption { + type = lib.types.str; + default = "http://127.0.0.1:${toString jackettCfg.port}"; + defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString cfg.jackett.port}"''; + description = "Jackett HTTP URL."; + }; + + jackettApiKeyFile = lib.mkOption { + type = lib.types.path; + description = "File containing the Jackett API key."; + }; + + qbittorrentUrl = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + qBittorrent WebUI URL override. + If null, auto-detected from VPN and port config. + ''; + }; + + qbittorrentPasswordFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = qbtCfg.passwordFile; + defaultText = lib.literalExpression "cfg.qbittorrent.passwordFile"; + description = "File containing qBittorrent WebUI password."; + }; + + indexerCache = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enable Jackett indexer cache with background refresh."; + }; + + refreshInterval = lib.mkOption { + type = lib.types.str; + default = "30m"; + description = "Background refresh interval."; + }; + + ttl = lib.mkOption { + type = lib.types.str; + default = "1h"; + description = "Cache entry time-to-live."; + }; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the gRPC port in the firewall."; + }; + }; + + config = lib.mkIf (cfg.enable && maCfg.enable) { + assertions = [ + { + assertion = metaCfg.enable; + message = "services.agregators.music-agregator requires services.agregators.metadata-agregator.enable"; + } + { + assertion = jackettCfg.enable || maCfg.jackettUrl != "http://127.0.0.1:${toString jackettCfg.port}"; + message = "services.agregators.music-agregator requires either jackett enabled or a custom jackettUrl"; + } + { + assertion = qbtCfg.enable || maCfg.qbittorrentUrl != null; + message = "services.agregators.music-agregator requires either qbittorrent enabled or a custom qbittorrentUrl"; + } + ]; + + systemd.services.music-agregator = { + description = "music-agregator - Music automation orchestrator"; + after = [ + "network.target" + "agregators-schema-music.service" + "metadata-agregator.service" + ] + ++ lib.optional jackettCfg.enable "jackett.service" + ++ lib.optional qbtCfg.enable "qbittorrent.service" + ++ lib.optional musicfsEnabled "musicfs.service"; + + requires = [ + "agregators-schema-music.service" + "metadata-agregator.service" + ]; + + wants = + lib.optional jackettCfg.enable "jackett.service" + ++ lib.optional qbtCfg.enable "qbittorrent.service" + ++ lib.optional musicfsEnabled "musicfs.service"; + + wantedBy = [ "multi-user.target" ]; + + preStart = '' + cp --no-preserve=mode ${configFile} /run/music-agregator/config.yaml + + # Inject database password + ${ + if pgCfg.musicDatabase.passwordFile != null then + '' + ${pkgs.replace-secret}/bin/replace-secret \ + '${dbPasswordPlaceholder}' \ + '${pgCfg.musicDatabase.passwordFile}' \ + /run/music-agregator/config.yaml + '' + else + '' + ${pkgs.gnused}/bin/sed -i "s/${dbPasswordPlaceholder}//" /run/music-agregator/config.yaml + '' + } + + # Inject Jackett API key + ${pkgs.replace-secret}/bin/replace-secret \ + '@JACKETT_API_KEY@' \ + '${maCfg.jackettApiKeyFile}' \ + /run/music-agregator/config.yaml + + # Inject qBittorrent password + ${ + if maCfg.qbittorrentPasswordFile != null then + '' + ${pkgs.replace-secret}/bin/replace-secret \ + '${qbtPasswordPlaceholder}' \ + '${maCfg.qbittorrentPasswordFile}' \ + /run/music-agregator/config.yaml + '' + else + '' + ${pkgs.gnused}/bin/sed -i "s/${qbtPasswordPlaceholder}//" /run/music-agregator/config.yaml + '' + } + ''; + + serviceConfig = { + Type = "simple"; + ExecStart = "${lib.getExe maCfg.package} -config /run/music-agregator/config.yaml"; + User = cfg.user; + Group = cfg.group; + RuntimeDirectory = "music-agregator"; + RuntimeDirectoryMode = "0750"; + Restart = "on-failure"; + RestartSec = 5; + + ProtectSystem = "strict"; + PrivateTmp = true; + NoNewPrivileges = true; + ProtectHome = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + }; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf maCfg.openFirewall [ maCfg.port ]; + }; +} diff --git a/nix/musicfs.nix b/nix/musicfs.nix new file mode 100644 index 0000000..95b070e --- /dev/null +++ b/nix/musicfs.nix @@ -0,0 +1,142 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; + mfsCfg = cfg.musicfs; + toml = pkgs.formats.toml { }; + + configFile = toml.generate "musicfs.toml" ( + { + mount_point = mfsCfg.mountPoint; + cache_dir = mfsCfg.cacheDir; + + origins = [ + { + id = mfsCfg.originId; + origin_type = "local"; + priority = 1; + enabled = true; + path = mfsCfg.originPath; + } + ]; + } + // lib.optionalAttrs (mfsCfg.cache != { }) { cache = mfsCfg.cache; } + // lib.optionalAttrs (mfsCfg.logging != { }) { logging = mfsCfg.logging; } + // mfsCfg.extraConfig + ); +in +{ + options.services.agregators.musicfs = { + enable = lib.mkEnableOption "musicfs virtual FUSE filesystem"; + + package = lib.mkOption { + type = lib.types.package; + description = "The musicfs package."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 50052; + description = "gRPC control API port."; + }; + + mountPoint = lib.mkOption { + type = lib.types.path; + default = "/mnt/music"; + description = "Where to mount the virtual filesystem."; + }; + + cacheDir = lib.mkOption { + type = lib.types.path; + default = "${cfg.stateDir}/musicfs/cache"; + defaultText = lib.literalExpression ''"''${cfg.stateDir}/musicfs/cache"''; + description = "Directory for cache data (CAS chunks, metadata, search index)."; + }; + + originPath = lib.mkOption { + type = lib.types.path; + default = "${cfg.mediaDir}/downloads"; + defaultText = lib.literalExpression ''"''${cfg.mediaDir}/downloads"''; + description = "Source directory for music files (typically qBittorrent download dir)."; + }; + + originId = lib.mkOption { + type = lib.types.str; + default = "local-storage"; + description = "Origin identifier used by musicfs and music-agregator."; + }; + + cache = lib.mkOption { + type = lib.types.attrsOf toml.type; + default = { }; + description = "Cache settings passed to [cache] in config."; + }; + + logging = lib.mkOption { + type = lib.types.attrsOf toml.type; + default = { }; + description = "Logging settings passed to [logging] in config."; + }; + + extraConfig = lib.mkOption { + type = lib.types.attrsOf toml.type; + default = { }; + description = "Additional top-level config keys."; + }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the gRPC port in the firewall."; + }; + }; + + config = lib.mkIf (cfg.enable && mfsCfg.enable) { + programs.fuse.userAllowOther = true; + + systemd.tmpfiles.rules = [ + "d '${mfsCfg.mountPoint}' 0755 ${cfg.user} ${cfg.group} - -" + "d '${mfsCfg.cacheDir}' 0750 ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.musicfs = { + description = "MusicFS - Virtual FUSE Filesystem for Music"; + after = [ + "network.target" + "local-fs.target" + ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "notify"; + ExecStart = "${lib.getExe mfsCfg.package} mount --config ${configFile} --grpc-port ${toString mfsCfg.port}"; + ExecStopPost = "${pkgs.fuse3}/bin/fusermount3 -u ${mfsCfg.mountPoint}"; + User = cfg.user; + Group = cfg.group; + Restart = "on-failure"; + RestartSec = 5; + + ProtectSystem = "strict"; + ReadWritePaths = [ + mfsCfg.mountPoint + mfsCfg.cacheDir + mfsCfg.originPath + ]; + PrivateTmp = true; + NoNewPrivileges = true; + ProtectHome = "read-only"; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + DeviceAllow = [ "/dev/fuse rw" ]; + }; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf mfsCfg.openFirewall [ mfsCfg.port ]; + }; +} diff --git a/nix/postgres.nix b/nix/postgres.nix new file mode 100644 index 0000000..4d0b8ae --- /dev/null +++ b/nix/postgres.nix @@ -0,0 +1,237 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; + pgCfg = cfg.postgres; + + # For local: run as the db user via peer auth (service runs as that user) + # For remote: run as anyone, use password auth via PGPASSWORD + mkPsql = + db: + lib.concatStringsSep " " ( + [ + "${pkgs.postgresql}/bin/psql" + ] + ++ lib.optionals (!isLocal) [ + "-h ${lib.escapeShellArg pgCfg.host}" + "-p ${toString pgCfg.port}" + "-U ${lib.escapeShellArg db.user}" + ] + ++ [ + "-d ${lib.escapeShellArg db.name}" + ] + ); + + mkPasswordExport = + db: + if db.passwordFile != null then + ''export PGPASSWORD="$(cat ${lib.escapeShellArg db.passwordFile})"'' + else + ""; + + isLocal = pgCfg.host == "/run/postgresql" || pgCfg.host == "localhost" || pgCfg.host == "127.0.0.1"; +in +{ + options.services.agregators.postgres = { + host = lib.mkOption { + type = lib.types.str; + default = "/run/postgresql"; + description = '' + PostgreSQL host. Use a path for unix socket (e.g. /run/postgresql) + or a hostname for TCP. + ''; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 5432; + description = "PostgreSQL port (ignored for unix socket connections)."; + }; + + musicDatabase = { + name = lib.mkOption { + type = lib.types.str; + default = "music_agregator"; + description = "Database name for music-agregator."; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "music_agregator"; + description = "Database user for music-agregator."; + }; + + passwordFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "Password file for music-agregator database. Null for peer auth."; + }; + }; + + metadataDatabase = { + name = lib.mkOption { + type = lib.types.str; + default = "metadata_agregator"; + description = "Database name for metadata-agregator."; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "metadata_agregator"; + description = "Database user for metadata-agregator."; + }; + + passwordFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "Password file for metadata-agregator database. Null for peer auth."; + }; + }; + + settings = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + description = "Extra PostgreSQL settings merged into services.postgresql.settings."; + example = { + shared_buffers = "256MB"; + effective_cache_size = "768MB"; + }; + }; + }; + + config = lib.mkIf cfg.enable { + # Create system users for peer auth (local postgres only) + users.users = lib.mkIf isLocal { + ${pgCfg.musicDatabase.user} = { + isSystemUser = true; + group = cfg.group; + }; + ${pgCfg.metadataDatabase.user} = { + isSystemUser = true; + group = cfg.group; + }; + }; + + # Enable and configure local PostgreSQL + services.postgresql = lib.mkIf isLocal { + enable = lib.mkDefault true; + settings = { + shared_preload_libraries = "pg_prewarm"; + } + // pgCfg.settings; + + ensureDatabases = [ + pgCfg.musicDatabase.name + pgCfg.metadataDatabase.name + ]; + + ensureUsers = [ + { + name = pgCfg.musicDatabase.user; + ensureDBOwnership = true; + } + { + name = pgCfg.metadataDatabase.user; + ensureDBOwnership = true; + } + ]; + }; + + # Schema init for metadata_agregator + systemd.services.agregators-schema-metadata = { + description = "Initialize metadata_agregator database schema"; + after = [ "network.target" ] ++ lib.optionals isLocal [ "postgresql.service" ]; + requires = lib.optionals isLocal [ "postgresql.service" ]; + before = [ "metadata-agregator.service" ]; + requiredBy = [ "metadata-agregator.service" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # Run as the db user for peer auth on local postgres + User = if isLocal then pgCfg.metadataDatabase.user else cfg.user; + }; + + script = + let + psql = mkPsql pgCfg.metadataDatabase; + passwordExport = mkPasswordExport pgCfg.metadataDatabase; + in + '' + ${passwordExport} + MARKER="${cfg.stateDir}/postgres/.schema-metadata-initialized" + + # Wait for database + for i in $(seq 1 30); do + if ${psql} -c "SELECT 1" >/dev/null 2>&1; then break; fi + echo "Waiting for metadata database... ($i/30)" + sleep 1 + done + + if [ -f "$MARKER" ]; then + echo "metadata_agregator schema already initialized" + exit 0 + fi + + echo "Applying metadata_agregator schema..." + ${psql} -f ${./sql/metadata/001_schema.sql} + + touch "$MARKER" + echo "metadata_agregator schema initialized" + ''; + }; + + # Schema init for music_agregator + systemd.services.agregators-schema-music = { + description = "Initialize music_agregator database schema"; + after = [ "network.target" ] ++ lib.optionals isLocal [ "postgresql.service" ]; + requires = lib.optionals isLocal [ "postgresql.service" ]; + before = [ "music-agregator.service" ]; + requiredBy = [ "music-agregator.service" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + # Run as the db user for peer auth on local postgres + User = if isLocal then pgCfg.musicDatabase.user else cfg.user; + }; + + script = + let + psql = mkPsql pgCfg.musicDatabase; + passwordExport = mkPasswordExport pgCfg.musicDatabase; + in + '' + ${passwordExport} + MARKER="${cfg.stateDir}/postgres/.schema-music-initialized" + + # Wait for database + for i in $(seq 1 30); do + if ${psql} -c "SELECT 1" >/dev/null 2>&1; then break; fi + echo "Waiting for music database... ($i/30)" + sleep 1 + done + + if [ -f "$MARKER" ]; then + echo "music_agregator schema already initialized" + exit 0 + fi + + echo "Applying music_agregator schema..." + ${psql} -f ${./sql/music/001_river.sql} + ${psql} -f ${./sql/music/002_schema.sql} + ${psql} -f ${./sql/music/003_event_bus.sql} + + touch "$MARKER" + echo "music_agregator schema initialized" + ''; + }; + }; +} diff --git a/nix/qbittorrent.nix b/nix/qbittorrent.nix new file mode 100644 index 0000000..912d73c --- /dev/null +++ b/nix/qbittorrent.nix @@ -0,0 +1,136 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.agregators; + qbtCfg = cfg.qbittorrent; + vpnCfg = cfg.vpn; + vpnEnabled = vpnCfg.enable && qbtCfg.vpn.enable; + + # Inside VPN namespace, services bind to the namespace address + namespaceAddr = "192.168.15.1"; + listenAddr = if vpnEnabled then namespaceAddr else "0.0.0.0"; +in +{ + options.services.agregators.qbittorrent = { + enable = lib.mkEnableOption "qBittorrent torrent client"; + + vpn = { + enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Confine qBittorrent to the VPN network namespace."; + }; + }; + + webuiPort = lib.mkOption { + type = lib.types.port; + default = 8080; + description = "qBittorrent WebUI port."; + }; + + peerPort = lib.mkOption { + type = lib.types.port; + default = 6881; + description = "BitTorrent peer port (opened through VPN)."; + }; + + downloadDir = lib.mkOption { + type = lib.types.path; + default = "${cfg.mediaDir}/downloads"; + defaultText = lib.literalExpression ''"''${cfg.mediaDir}/downloads"''; + description = "Directory where qBittorrent saves completed downloads."; + }; + + username = lib.mkOption { + type = lib.types.str; + default = "admin"; + description = "qBittorrent WebUI username."; + }; + + passwordFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "File containing qBittorrent WebUI password."; + }; + + stateDir = lib.mkOption { + type = lib.types.path; + default = "${cfg.stateDir}/qbittorrent"; + defaultText = lib.literalExpression ''"''${cfg.stateDir}/qbittorrent"''; + description = "qBittorrent state/config directory."; + }; + + package = lib.mkPackageOption pkgs "qbittorrent-nox" { }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Open the WebUI port in the firewall."; + }; + }; + + config = lib.mkIf (cfg.enable && qbtCfg.enable) { + assertions = [ + { + assertion = qbtCfg.vpn.enable -> vpnCfg.enable; + message = "services.agregators.qbittorrent.vpn.enable requires services.agregators.vpn.enable"; + } + ]; + + systemd.tmpfiles.rules = [ + "d '${qbtCfg.stateDir}' 0750 ${cfg.user} ${cfg.group} - -" + "d '${qbtCfg.stateDir}/qBittorrent' 0750 ${cfg.user} ${cfg.group} - -" + "d '${qbtCfg.stateDir}/qBittorrent/config' 0750 ${cfg.user} ${cfg.group} - -" + "d '${qbtCfg.downloadDir}' 0775 ${cfg.user} ${cfg.group} - -" + "d '${qbtCfg.downloadDir}/.incomplete' 0775 ${cfg.user} ${cfg.group} - -" + ]; + + systemd.services.qbittorrent = { + description = "qBittorrent-nox BitTorrent client"; + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Type = "simple"; + User = cfg.user; + Group = cfg.group; + ExecStart = lib.concatStringsSep " " [ + "${lib.getExe qbtCfg.package}" + "--webui-port=${toString qbtCfg.webuiPort}" + "--profile=${qbtCfg.stateDir}" + ]; + Restart = "on-failure"; + RestartSec = 5; + }; + }; + + # VPN confinement + systemd.services.qbittorrent.vpnConfinement = lib.mkIf vpnEnabled { + enable = true; + vpnNamespace = vpnCfg.namespace; + }; + + vpnNamespaces.${vpnCfg.namespace} = lib.mkIf vpnEnabled { + portMappings = [ + { + from = qbtCfg.webuiPort; + to = qbtCfg.webuiPort; + } + ]; + openVPNPorts = [ + { + port = qbtCfg.peerPort; + protocol = "both"; + } + ]; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf qbtCfg.openFirewall [ qbtCfg.webuiPort ]; + }; +} diff --git a/nix/sql/metadata/001_schema.sql b/nix/sql/metadata/001_schema.sql new file mode 100644 index 0000000..3cf729c --- /dev/null +++ b/nix/sql/metadata/001_schema.sql @@ -0,0 +1,194 @@ +CREATE EXTENSION IF NOT EXISTS pg_prewarm; +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE TABLE artists ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + sort_name TEXT, + artist_type TEXT, + country TEXT, + formed_date DATE, + disbanded_date DATE, + description TEXT, + image_url TEXT, + source TEXT NOT NULL, + source_id TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE works ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + title TEXT NOT NULL, + work_type TEXT, + language TEXT, + source TEXT NOT NULL, + source_id TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE tracks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + work_id UUID REFERENCES works(id), + title TEXT NOT NULL, + duration_ms INT, + isrc TEXT, + explicit BOOLEAN DEFAULT false, + source TEXT NOT NULL, + source_id TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE labels ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + country TEXT, + founded_date DATE, + source TEXT NOT NULL, + source_id TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE albums ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + label_id UUID REFERENCES labels(id), + title TEXT NOT NULL, + album_type TEXT, + secondary_types TEXT[] DEFAULT '{}', + release_date DATE, + upc TEXT, + total_tracks INT, + total_discs INT DEFAULT 1, + cover_url TEXT, + source TEXT NOT NULL, + source_id TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE genres ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + parent_id UUID REFERENCES genres(id) +); + +CREATE TABLE track_artists ( + track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, + artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + role TEXT DEFAULT 'primary', + position INT DEFAULT 0, + PRIMARY KEY (track_id, artist_id, role) +); + +CREATE TABLE album_artists ( + album_id UUID REFERENCES albums(id) ON DELETE CASCADE, + artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + role TEXT DEFAULT 'primary', + position INT DEFAULT 0, + PRIMARY KEY (album_id, artist_id, role) +); + +CREATE TABLE album_tracks ( + album_id UUID REFERENCES albums(id) ON DELETE CASCADE, + track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, + disc_number INT DEFAULT 1, + track_number INT NOT NULL, + PRIMARY KEY (album_id, track_id) +); + +CREATE TABLE work_artists ( + work_id UUID REFERENCES works(id) ON DELETE CASCADE, + artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + role TEXT DEFAULT 'writer', + PRIMARY KEY (work_id, artist_id, role) +); + +CREATE TABLE artist_genres ( + artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + genre_id UUID REFERENCES genres(id) ON DELETE CASCADE, + PRIMARY KEY (artist_id, genre_id) +); + +CREATE TABLE album_genres ( + album_id UUID REFERENCES albums(id) ON DELETE CASCADE, + genre_id UUID REFERENCES genres(id) ON DELETE CASCADE, + PRIMARY KEY (album_id, genre_id) +); + +CREATE TABLE similar_artists ( + artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + similar_artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + score REAL DEFAULT 0.5, + PRIMARY KEY (artist_id, similar_artist_id) +); + +CREATE TABLE lyrics ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, + content TEXT, + synced_content JSONB, + language TEXT, + source TEXT NOT NULL, + source_id TEXT, + created_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE playlists ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + description TEXT, + is_public BOOLEAN DEFAULT true, + cover_url TEXT, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE TABLE playlist_tracks ( + playlist_id UUID REFERENCES playlists(id) ON DELETE CASCADE, + track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, + position INT NOT NULL, + added_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (playlist_id, track_id) +); + +CREATE TABLE artist_external_ids ( + artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, + source TEXT NOT NULL, + source_id TEXT NOT NULL, + url TEXT, + fetched_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (artist_id, source, source_id) +); + +CREATE TABLE album_external_ids ( + album_id UUID REFERENCES albums(id) ON DELETE CASCADE, + source TEXT NOT NULL, + source_id TEXT NOT NULL, + url TEXT, + fetched_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (album_id, source, source_id) +); + +CREATE TABLE track_external_ids ( + track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, + source TEXT NOT NULL, + source_id TEXT NOT NULL, + url TEXT, + fetched_at TIMESTAMPTZ DEFAULT now(), + PRIMARY KEY (track_id, source, source_id) +); + +CREATE INDEX idx_artists_name ON artists(name); +CREATE INDEX idx_artists_name_trgm ON artists USING gin (name gin_trgm_ops); +CREATE INDEX idx_artists_source ON artists(source, source_id); +CREATE INDEX idx_tracks_isrc ON tracks(isrc) WHERE isrc IS NOT NULL; +CREATE INDEX idx_tracks_source ON tracks(source, source_id); +CREATE INDEX idx_albums_upc ON albums(upc) WHERE upc IS NOT NULL; +CREATE INDEX idx_albums_source ON albums(source, source_id); +CREATE INDEX idx_albums_release_date ON albums(release_date); +CREATE INDEX idx_genres_name ON genres(name); +CREATE INDEX idx_lyrics_track_id ON lyrics(track_id); +CREATE INDEX idx_playlist_tracks_position ON playlist_tracks(playlist_id, position); diff --git a/nix/sql/music/001_river.sql b/nix/sql/music/001_river.sql new file mode 100644 index 0000000..2319355 --- /dev/null +++ b/nix/sql/music/001_river.sql @@ -0,0 +1,286 @@ +-- Version 1 +CREATE TABLE river_migration( + id bigserial PRIMARY KEY, + created_at timestamptz NOT NULL DEFAULT NOW(), + version bigint NOT NULL, + CONSTRAINT version CHECK (version >= 1) +); + +CREATE UNIQUE INDEX ON river_migration USING btree(version); + +-- Version 2 +CREATE TYPE river_job_state AS ENUM( + 'available', + 'cancelled', + 'completed', + 'discarded', + 'retryable', + 'running', + 'scheduled' +); + +CREATE TABLE river_job( + -- 8 bytes + id bigserial PRIMARY KEY, + + -- 8 bytes (4 bytes + 2 bytes + 2 bytes) + -- + -- `state` is kept near the top of the table for operator convenience -- when + -- looking at jobs with `SELECT *` it'll appear first after ID. The other two + -- fields aren't as important but are kept adjacent to `state` for alignment + -- to get an 8-byte block. + state river_job_state NOT NULL DEFAULT 'available', + attempt smallint NOT NULL DEFAULT 0, + max_attempts smallint NOT NULL, + + -- 8 bytes each (no alignment needed) + attempted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT NOW(), + finalized_at timestamptz, + scheduled_at timestamptz NOT NULL DEFAULT NOW(), + + -- 2 bytes (some wasted padding probably) + priority smallint NOT NULL DEFAULT 1, + + -- types stored out-of-band + args jsonb, + attempted_by text[], + errors jsonb[], + kind text NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}', + queue text NOT NULL DEFAULT 'default', + tags varchar(255)[], + + CONSTRAINT finalized_or_finalized_at_null CHECK ((state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR finalized_at IS NULL), + CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), + CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), + CONSTRAINT queue_length CHECK (char_length(queue) > 0 AND char_length(queue) < 128), + CONSTRAINT kind_length CHECK (char_length(kind) > 0 AND char_length(kind) < 128) +); + +-- We may want to consider adding another property here after `kind` if it seems +-- like it'd be useful for something. +CREATE INDEX river_job_kind ON river_job USING btree(kind); + +CREATE INDEX river_job_state_and_finalized_at_index ON river_job USING btree(state, finalized_at) WHERE finalized_at IS NOT NULL; + +CREATE INDEX river_job_prioritized_fetching_index ON river_job USING btree(state, queue, priority, scheduled_at, id); + +CREATE INDEX river_job_args_index ON river_job USING GIN(args); + +CREATE INDEX river_job_metadata_index ON river_job USING GIN(metadata); + +CREATE OR REPLACE FUNCTION river_job_notify() + RETURNS TRIGGER + AS $$ +DECLARE + payload json; +BEGIN + IF NEW.state = 'available' THEN + -- Notify will coalesce duplicate notifications within a transaction, so + -- keep these payloads generalized: + payload = json_build_object('queue', NEW.queue); + PERFORM + pg_notify('river_insert', payload::text); + END IF; + RETURN NULL; +END; +$$ +LANGUAGE plpgsql; + +CREATE TRIGGER river_notify + AFTER INSERT ON river_job + FOR EACH ROW + EXECUTE PROCEDURE river_job_notify(); + +CREATE UNLOGGED TABLE river_leader( + -- 8 bytes each (no alignment needed) + elected_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + + -- types stored out-of-band + leader_id text NOT NULL, + name text PRIMARY KEY, + + CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128), + CONSTRAINT leader_id_length CHECK (char_length(leader_id) > 0 AND char_length(leader_id) < 128) +); + + +-- Version 3 +ALTER TABLE river_job ALTER COLUMN tags SET DEFAULT '{}'; +UPDATE river_job SET tags = '{}' WHERE tags IS NULL; +ALTER TABLE river_job ALTER COLUMN tags SET NOT NULL; + + +-- Version 4 +-- The args column never had a NOT NULL constraint or default value at the +-- database level, though we tried to ensure one at the application level. +ALTER TABLE river_job ALTER COLUMN args SET DEFAULT '{}'; +UPDATE river_job SET args = '{}' WHERE args IS NULL; +ALTER TABLE river_job ALTER COLUMN args SET NOT NULL; +ALTER TABLE river_job ALTER COLUMN args DROP DEFAULT; + +-- The metadata column never had a NOT NULL constraint or default value at the +-- database level, though we tried to ensure one at the application level. +ALTER TABLE river_job ALTER COLUMN metadata SET DEFAULT '{}'; +UPDATE river_job SET metadata = '{}' WHERE metadata IS NULL; +ALTER TABLE river_job ALTER COLUMN metadata SET NOT NULL; + +-- The 'pending' job state will be used for upcoming functionality: +ALTER TYPE river_job_state ADD VALUE IF NOT EXISTS 'pending' AFTER 'discarded'; + +ALTER TABLE river_job DROP CONSTRAINT finalized_or_finalized_at_null; +ALTER TABLE river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK ( + (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR + (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) +); + +DROP TRIGGER river_notify ON river_job; +DROP FUNCTION river_job_notify; + +-- +-- Create table `river_queue`. +-- + +CREATE TABLE river_queue ( + name text PRIMARY KEY NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + metadata jsonb NOT NULL DEFAULT '{}' ::jsonb, + paused_at timestamptz, + updated_at timestamptz NOT NULL +); + +-- +-- Alter `river_leader` to add a default value of 'default` to `name`. +-- + +ALTER TABLE river_leader + ALTER COLUMN name SET DEFAULT 'default', + DROP CONSTRAINT name_length, + ADD CONSTRAINT name_length CHECK (name = 'default'); + +-- Version 5 +-- +-- Rebuild the migration table so it's based on `(line, version)`. +-- + +DO +$body$ +BEGIN + -- Tolerate users who may be using their own migration system rather than + -- River's. If they are, they will have skipped version 001 containing + -- `CREATE TABLE river_migration`, so this table won't exist. + IF (SELECT to_regclass('river_migration') IS NOT NULL) THEN + ALTER TABLE river_migration + RENAME TO river_migration_old; + + CREATE TABLE river_migration( + line TEXT NOT NULL, + version bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT NOW(), + CONSTRAINT line_length CHECK (char_length(line) > 0 AND char_length(line) < 128), + CONSTRAINT version_gte_1 CHECK (version >= 1), + PRIMARY KEY (line, version) + ); + + INSERT INTO river_migration + (created_at, line, version) + SELECT created_at, 'main', version + FROM river_migration_old; + + DROP TABLE river_migration_old; + END IF; +END; +$body$ +LANGUAGE 'plpgsql'; + +-- +-- Add `river_job.unique_key` and bring up an index on it. +-- + +-- These statements use `IF NOT EXISTS` to allow users with a `river_job` table +-- of non-trivial size to build the index `CONCURRENTLY` out of band of this +-- migration, then follow by completing the migration. +ALTER TABLE river_job + ADD COLUMN IF NOT EXISTS unique_key bytea; + +CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON river_job (kind, unique_key) WHERE unique_key IS NOT NULL; + +-- +-- Create `river_client` and derivative. +-- +-- This feature hasn't quite yet been implemented, but we're taking advantage of +-- the migration to add the schema early so that we can add it later without an +-- additional migration. +-- + +CREATE UNLOGGED TABLE river_client ( + id text PRIMARY KEY NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + metadata jsonb NOT NULL DEFAULT '{}', + paused_at timestamptz, + updated_at timestamptz NOT NULL, + CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128) +); + +-- Differs from `river_queue` in that it tracks the queue state for a particular +-- active client. +CREATE UNLOGGED TABLE river_client_queue ( + river_client_id text NOT NULL REFERENCES river_client (id) ON DELETE CASCADE, + name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + max_workers bigint NOT NULL DEFAULT 0, + metadata jsonb NOT NULL DEFAULT '{}', + num_jobs_completed bigint NOT NULL DEFAULT 0, + num_jobs_running bigint NOT NULL DEFAULT 0, + updated_at timestamptz NOT NULL, + PRIMARY KEY (river_client_id, name), + CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128), + CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), + CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) +); + +-- Version 6 +CREATE OR REPLACE FUNCTION river_job_state_in_bitmask(bitmask BIT(8), state river_job_state) +RETURNS boolean +LANGUAGE SQL +IMMUTABLE +AS $$ + SELECT CASE state + WHEN 'available' THEN get_bit(bitmask, 7) + WHEN 'cancelled' THEN get_bit(bitmask, 6) + WHEN 'completed' THEN get_bit(bitmask, 5) + WHEN 'discarded' THEN get_bit(bitmask, 4) + WHEN 'pending' THEN get_bit(bitmask, 3) + WHEN 'retryable' THEN get_bit(bitmask, 2) + WHEN 'running' THEN get_bit(bitmask, 1) + WHEN 'scheduled' THEN get_bit(bitmask, 0) + ELSE 0 + END = 1; +$$; + +-- +-- Add `river_job.unique_states` and bring up an index on it. +-- +-- This column may exist already if users manually created the column and index +-- as instructed in the changelog so the index could be created `CONCURRENTLY`. +-- +ALTER TABLE river_job ADD COLUMN IF NOT EXISTS unique_states BIT(8); + +-- This statement uses `IF NOT EXISTS` to allow users with a `river_job` table +-- of non-trivial size to build the index `CONCURRENTLY` out of band of this +-- migration, then follow by completing the migration. +CREATE UNIQUE INDEX IF NOT EXISTS river_job_unique_idx ON river_job (unique_key) + WHERE unique_key IS NOT NULL + AND unique_states IS NOT NULL + AND river_job_state_in_bitmask(unique_states, state); + +-- Remove the old unique index. Users who are actively using the unique jobs +-- feature and who wish to avoid deploy downtime may want od drop this in a +-- subsequent migration once all jobs using the old unique system have been +-- completed (i.e. no more rows with non-null unique_key and null +-- unique_states). +DROP INDEX river_job_kind_unique_key_idx; + + diff --git a/nix/sql/music/002_schema.sql b/nix/sql/music/002_schema.sql new file mode 100644 index 0000000..0e41ea3 --- /dev/null +++ b/nix/sql/music/002_schema.sql @@ -0,0 +1,166 @@ +CREATE TYPE monitor_state AS ENUM ('monitored', 'unmonitored', 'excluded'); + +CREATE TABLE artists ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id VARCHAR(255) NOT NULL UNIQUE, + name VARCHAR(500) NOT NULL, + artist_type VARCHAR(50), + country VARCHAR(10), + genres TEXT[], + image_url TEXT, + monitor_state monitor_state NOT NULL DEFAULT 'unmonitored', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE albums ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id VARCHAR(255) NOT NULL UNIQUE, + artist_id UUID NOT NULL REFERENCES artists(id) ON DELETE CASCADE, + title VARCHAR(500) NOT NULL, + album_type VARCHAR(50), + release_date DATE, + total_tracks INT, + total_discs INT DEFAULT 1, + label VARCHAR(255), + genres TEXT[], + cover_url TEXT, + monitor_state monitor_state NOT NULL DEFAULT 'unmonitored', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_albums_artist ON albums(artist_id); +CREATE INDEX idx_albums_monitored ON albums(monitor_state) WHERE monitor_state = 'monitored'; + +CREATE TABLE tracks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_id VARCHAR(255) NOT NULL UNIQUE, + album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE, + title VARCHAR(500) NOT NULL, + duration_ms INT, + isrc VARCHAR(20), + disc_number INT DEFAULT 1, + track_number INT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_tracks_album ON tracks(album_id); + +CREATE TABLE torrents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE, + info_hash VARCHAR(40) NOT NULL UNIQUE, + tracker VARCHAR(100) NOT NULL, + title TEXT NOT NULL, + format VARCHAR(20) NOT NULL, + quality VARCHAR(20), + source VARCHAR(20), + bit_depth INT, + sample_rate INT, + seeders INT DEFAULT 0, + peers INT DEFAULT 0, + size BIGINT, + track_count INT, + has_cover_art BOOLEAN DEFAULT FALSE, + has_cue_sheet BOOLEAN DEFAULT FALSE, + has_rip_log BOOLEAN DEFAULT FALSE, + download_link TEXT, + torrent_file BYTEA, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_torrents_album ON torrents(album_id); +CREATE INDEX idx_torrents_album_format ON torrents(album_id, format); + +CREATE TYPE download_state AS ENUM ( + 'pending', 'downloading', 'completed', 'failed', 'seeding', 'paused' +); + +CREATE TABLE downloads ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + torrent_id UUID NOT NULL REFERENCES torrents(id) ON DELETE CASCADE, + album_id UUID NOT NULL, + format VARCHAR(20) NOT NULL, + quality VARCHAR(20), + state download_state NOT NULL DEFAULT 'pending', + qbit_hash VARCHAR(64), + save_path TEXT, + error_message TEXT, + queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE UNIQUE INDEX idx_unique_active_torrent + ON downloads(torrent_id) + WHERE state NOT IN ('failed'); + +CREATE UNIQUE INDEX idx_unique_owned_album_quality + ON downloads(album_id, format, quality) + WHERE state IN ('completed', 'seeding'); + +CREATE INDEX idx_downloads_state ON downloads(state); +CREATE INDEX idx_downloads_album ON downloads(album_id); + +CREATE TABLE download_files ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + download_id UUID NOT NULL REFERENCES downloads(id) ON DELETE CASCADE, + track_id UUID REFERENCES tracks(id), + file_path TEXT NOT NULL, + file_size BIGINT NOT NULL, + file_type VARCHAR(20) NOT NULL, + sha256_hash VARCHAR(64), + verified_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_download_files_download ON download_files(download_id); +CREATE INDEX idx_download_files_hash ON download_files(sha256_hash) WHERE sha256_hash IS NOT NULL; + +CREATE TABLE album_releases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE, + download_id UUID NOT NULL REFERENCES downloads(id) ON DELETE CASCADE, + format VARCHAR(20) NOT NULL, + bit_depth INT, + sample_rate INT, + channels INT DEFAULT 2, + is_lossless BOOLEAN NOT NULL DEFAULT FALSE, + source VARCHAR(20), + total_size BIGINT NOT NULL DEFAULT 0, + total_duration_ms INT NOT NULL DEFAULT 0, + track_count INT NOT NULL DEFAULT 0, + has_cover_art BOOLEAN DEFAULT FALSE, + has_cue_sheet BOOLEAN DEFAULT FALSE, + has_rip_log BOOLEAN DEFAULT FALSE, + path TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_album_releases_album ON album_releases(album_id); +CREATE UNIQUE INDEX idx_album_releases_download ON album_releases(download_id); + +CREATE TABLE track_releases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + album_release_id UUID NOT NULL REFERENCES album_releases(id) ON DELETE CASCADE, + track_id UUID REFERENCES tracks(id), + download_file_id UUID REFERENCES download_files(id), + title VARCHAR(500) NOT NULL, + track_number INT NOT NULL DEFAULT 1, + disc_number INT NOT NULL DEFAULT 1, + duration_ms INT, + format VARCHAR(20) NOT NULL, + bit_depth INT, + sample_rate INT, + channels INT DEFAULT 2, + bitrate_kbps INT, + file_size BIGINT NOT NULL DEFAULT 0, + file_path TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_track_releases_album_release ON track_releases(album_release_id); diff --git a/nix/sql/music/003_event_bus.sql b/nix/sql/music/003_event_bus.sql new file mode 100644 index 0000000..6e8ec7c --- /dev/null +++ b/nix/sql/music/003_event_bus.sql @@ -0,0 +1,33 @@ +CREATE TABLE workflow_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE, + quality VARCHAR(20) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'running', + error_message TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + running_lock BOOLEAN GENERATED ALWAYS AS (CASE WHEN status = 'running' THEN TRUE ELSE NULL END) STORED, + CONSTRAINT idx_workflow_runs_active UNIQUE (album_id, quality, running_lock) +); + +CREATE INDEX idx_workflow_runs_status ON workflow_runs(status); + +CREATE TABLE album_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + seq BIGSERIAL NOT NULL, + workflow_run_id UUID NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE, + album_id UUID NOT NULL, + event_type VARCHAR(20) NOT NULL, + step VARCHAR(50) NOT NULL, + message TEXT NOT NULL, + data_json JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_album_events_workflow ON album_events(workflow_run_id); +CREATE INDEX idx_album_events_album ON album_events(album_id); +CREATE INDEX idx_album_events_seq ON album_events(seq); + +ALTER TYPE download_state ADD VALUE IF NOT EXISTS 'cancelled'; diff --git a/nix/vpn.nix b/nix/vpn.nix new file mode 100644 index 0000000..f9e122d --- /dev/null +++ b/nix/vpn.nix @@ -0,0 +1,48 @@ +{ + config, + lib, + ... +}: + +let + cfg = config.services.agregators; + vpnCfg = cfg.vpn; +in +{ + options.services.agregators.vpn = { + enable = lib.mkEnableOption "WireGuard VPN via VPN-Confinement for torrent traffic"; + + wgConfigFile = lib.mkOption { + type = lib.types.path; + description = '' + Path to WireGuard configuration file. + Must not be in the Nix store — use a runtime path. + ''; + example = "/run/secrets/wireguard.conf"; + }; + + accessibleFrom = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ + "192.168.1.0/24" + "192.168.0.0/24" + "127.0.0.1" + ]; + description = "CIDRs allowed to reach services inside the VPN namespace."; + }; + + namespace = lib.mkOption { + type = lib.types.str; + default = "wg"; + description = "Name of the VPN network namespace."; + }; + }; + + config = lib.mkIf (cfg.enable && vpnCfg.enable) { + vpnNamespaces.${vpnCfg.namespace} = { + enable = true; + wireguardConfigFile = vpnCfg.wgConfigFile; + accessibleFrom = vpnCfg.accessibleFrom; + }; + }; +} diff --git a/tests/simple-test.nix b/tests/simple-test.nix new file mode 100644 index 0000000..a54ae7e --- /dev/null +++ b/tests/simple-test.nix @@ -0,0 +1,104 @@ +{ + pkgs, + nixosModules, +}: + +pkgs.testers.nixosTest { + name = "agregators-simple-test"; + + nodes.machine = + { + config, + pkgs, + ... + }: + { + imports = [ nixosModules.default ]; + + networking.firewall.enable = false; + + virtualisation.memorySize = 4096; + virtualisation.diskSize = 8192; + + services.agregators = { + enable = true; + mediaDir = "/data/music"; + + # No VPN in test — no WireGuard endpoint to connect to + # qbittorrent.vpn.enable is false by implication + + qbittorrent = { + enable = true; + vpn.enable = false; + }; + + jackett.enable = true; + + metadata-agregator = { + enable = true; + package = config.services.agregators.music-agregator.package; + # In real usage this would be the actual metadata-agregator package. + # For the simple test we just check the module evaluates and + # systemd units are created. The service itself won't start + # without the real binary — that's fine, we test structure here. + }; + + musicfs = { + enable = true; + package = config.services.agregators.music-agregator.package; + }; + + music-agregator = { + enable = true; + package = pkgs.hello; # placeholder — real package tested separately + jackettApiKeyFile = "/dev/null"; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + + # PostgreSQL should be running with both databases + machine.succeed("systemctl is-active postgresql") + machine.succeed("sudo -u postgres psql -lqt | grep -q music_agregator") + machine.succeed("sudo -u postgres psql -lqt | grep -q metadata_agregator") + + # Schema init oneshots should have completed + machine.succeed("systemctl is-active agregators-schema-music") + machine.succeed("systemctl is-active agregators-schema-metadata") + + # Verify schemas were applied (marker files exist) + machine.succeed("test -f /var/lib/agregators/postgres/.schema-music-initialized") + machine.succeed("test -f /var/lib/agregators/postgres/.schema-metadata-initialized") + + # Verify tables exist in music_agregator + machine.succeed("sudo -u music_agregator psql -d music_agregator -c 'SELECT 1 FROM river_job LIMIT 0'") + machine.succeed("sudo -u music_agregator psql -d music_agregator -c 'SELECT 1 FROM artists LIMIT 0'") + machine.succeed("sudo -u music_agregator psql -d music_agregator -c 'SELECT 1 FROM workflow_runs LIMIT 0'") + + # Verify tables exist in metadata_agregator + machine.succeed("sudo -u metadata_agregator psql -d metadata_agregator -c 'SELECT 1 FROM artists LIMIT 0'") + machine.succeed("sudo -u metadata_agregator psql -d metadata_agregator -c 'SELECT 1 FROM albums LIMIT 0'") + + # Verify extensions + machine.succeed("sudo -u metadata_agregator psql -d metadata_agregator -c \"SELECT 1 FROM pg_extension WHERE extname='pg_trgm'\" | grep -q 1") + + # qBittorrent should be running (no VPN in test) + machine.succeed("systemctl is-active qbittorrent") + + # Jackett should be running + machine.succeed("systemctl is-active jackett") + + # Directories should exist with correct ownership + machine.succeed("test -d /data/music") + machine.succeed("test -d /data/music/downloads") + machine.succeed("test -d /var/lib/agregators") + + # Verify the agregators user/group exist + machine.succeed("id agregators") + machine.succeed("getent group agregators") + + print("\n=== Agregators Simple Test Completed ===") + ''; +}