Add NixOS module for full agregators stack with VM test

Module manages PostgreSQL (both databases + schema init), qBittorrent
(VPN-confined via VPN-Confinement), Jackett, metadata-agregator,
musicfs, and the main orchestrator. All services are independently
enableable with auto-wired inter-service configuration.

Includes NixOS VM test validating PostgreSQL setup, schema
initialization, service startup, and directory creation.

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/claude-agent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
Alexander
2026-05-20 12:39:04 +02:00
parent 10a5fd5707
commit 908c37f73f
15 changed files with 1939 additions and 1 deletions
Generated
+17 -1
View File
@@ -125,7 +125,23 @@
"inputs": { "inputs": {
"flake-parts": "flake-parts", "flake-parts": "flake-parts",
"git-hooks": "git-hooks", "git-hooks": "git-hooks",
"nixpkgs": "nixpkgs_2" "nixpkgs": "nixpkgs_2",
"vpnconfinement": "vpnconfinement"
}
},
"vpnconfinement": {
"locked": {
"lastModified": 1778182451,
"narHash": "sha256-Bz3n2THDGf90Z9gMqhH/J492prYH8B6RFRlxv/fPBwc=",
"owner": "Maroka-chan",
"repo": "VPN-Confinement",
"rev": "cf5bfc4c3559f2e783698b1aa23c165072039a7d",
"type": "github"
},
"original": {
"owner": "Maroka-chan",
"repo": "VPN-Confinement",
"type": "github"
} }
} }
}, },
+16
View File
@@ -3,6 +3,7 @@
nixpkgs.url = "github:nixos/nixpkgs"; nixpkgs.url = "github:nixos/nixpkgs";
flake-parts.url = "github:hercules-ci/flake-parts"; flake-parts.url = "github:hercules-ci/flake-parts";
git-hooks.url = "github:cachix/git-hooks.nix"; git-hooks.url = "github:cachix/git-hooks.nix";
vpnconfinement.url = "github:Maroka-chan/VPN-Confinement";
}; };
outputs = outputs =
@@ -18,6 +19,18 @@
"x86_64-linux" "x86_64-linux"
]; ];
flake = {
nixosModules = {
agregators = {
imports = [
./nix/default.nix
inputs.vpnconfinement.nixosModules.default
];
};
default = self.nixosModules.agregators;
};
};
perSystem = perSystem =
{ {
system, system,
@@ -49,6 +62,9 @@
checks = { checks = {
inherit pre-commit-check; inherit pre-commit-check;
simple-test = pkgs.callPackage ./tests/simple-test.nix {
nixosModules = self.nixosModules;
};
}; };
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
+68
View File
@@ -0,0 +1,68 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
in
{
imports = [
./postgres.nix
./vpn.nix
./qbittorrent.nix
./jackett.nix
./metadata-agregator.nix
./musicfs.nix
./music-agregator.nix
];
options.services.agregators = {
enable = lib.mkEnableOption "agregators music automation pipeline";
mediaDir = lib.mkOption {
type = lib.types.path;
default = "/data/music";
description = ''
Root directory for music files.
qBittorrent downloads here, musicfs reads from here.
'';
};
stateDir = lib.mkOption {
type = lib.types.path;
default = "/var/lib/agregators";
description = "Root state directory. Per-service subdirectories are created automatically.";
};
user = lib.mkOption {
type = lib.types.str;
default = "agregators";
description = "Shared system user for all agregators services.";
};
group = lib.mkOption {
type = lib.types.str;
default = "agregators";
description = "Shared system group for all agregators services.";
};
};
config = lib.mkIf cfg.enable {
users.users.${cfg.user} = lib.mkIf (cfg.user == "agregators") {
isSystemUser = true;
group = cfg.group;
home = cfg.stateDir;
};
users.groups.${cfg.group} = lib.mkIf (cfg.group == "agregators") { };
systemd.tmpfiles.rules = [
"d '${cfg.mediaDir}' 0775 ${cfg.user} ${cfg.group} - -"
"d '${cfg.stateDir}' 0750 ${cfg.user} ${cfg.group} - -"
"d '${cfg.stateDir}/postgres' 0770 ${cfg.user} ${cfg.group} - -"
];
};
}
+75
View File
@@ -0,0 +1,75 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
jackettCfg = cfg.jackett;
in
{
options.services.agregators.jackett = {
enable = lib.mkEnableOption "Jackett torrent indexer aggregator";
port = lib.mkOption {
type = lib.types.port;
default = 9117;
description = "Jackett HTTP port.";
};
stateDir = lib.mkOption {
type = lib.types.path;
default = "${cfg.stateDir}/jackett";
defaultText = lib.literalExpression ''"''${cfg.stateDir}/jackett"'';
description = "Jackett data directory.";
};
apiKeyFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
File containing the Jackett API key.
If null, music-agregator must be configured with the API key manually.
'';
};
package = lib.mkPackageOption pkgs "jackett" { };
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Open the Jackett port in the firewall.";
};
};
config = lib.mkIf (cfg.enable && jackettCfg.enable) {
systemd.tmpfiles.rules = [
"d '${jackettCfg.stateDir}' 0750 ${cfg.user} ${cfg.group} - -"
];
systemd.services.jackett = {
description = "Jackett torrent indexer aggregator";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "simple";
User = cfg.user;
Group = cfg.group;
ExecStart = lib.concatStringsSep " " [
"${lib.getExe jackettCfg.package}"
"--DataFolder=${jackettCfg.stateDir}"
"--Port=${toString jackettCfg.port}"
"--NoUpdates"
];
Restart = "on-failure";
RestartSec = 5;
};
};
networking.firewall.allowedTCPPorts = lib.mkIf jackettCfg.openFirewall [ jackettCfg.port ];
};
}
+142
View File
@@ -0,0 +1,142 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
metaCfg = cfg.metadata-agregator;
pgCfg = cfg.postgres;
db = pgCfg.metadataDatabase;
yaml = pkgs.formats.yaml { };
dbPasswordPlaceholder = "@META_DB_PASSWORD@";
configFile = yaml.generate "metadata-agregator.yaml" {
server.port = metaCfg.port;
database = {
host = if pgCfg.host == "/run/postgresql" then "localhost" else pgCfg.host;
port = pgCfg.port;
name = db.name;
user = db.user;
password = dbPasswordPlaceholder;
sslmode = "disable";
};
logging = {
level = metaCfg.logging.level;
format = metaCfg.logging.format;
};
metrics = {
enabled = metaCfg.metrics.enable;
port = metaCfg.metrics.port;
};
};
in
{
options.services.agregators.metadata-agregator = {
enable = lib.mkEnableOption "metadata-agregator music metadata gRPC service";
package = lib.mkOption {
type = lib.types.package;
description = "The metadata-agregator package.";
};
port = lib.mkOption {
type = lib.types.port;
default = 50051;
description = "gRPC listen port.";
};
logging = {
level = lib.mkOption {
type = lib.types.str;
default = "info";
description = "Log level.";
};
format = lib.mkOption {
type = lib.types.enum [
"json"
"console"
];
default = "json";
description = "Log output format.";
};
};
metrics = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Enable Prometheus metrics endpoint.";
};
port = lib.mkOption {
type = lib.types.port;
default = 9091;
description = "Prometheus metrics HTTP port.";
};
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Open the gRPC port in the firewall.";
};
};
config = lib.mkIf (cfg.enable && metaCfg.enable) {
systemd.services.metadata-agregator = {
description = "metadata-agregator - Music metadata gRPC service";
after = [
"network.target"
"agregators-schema-metadata.service"
];
requires = [ "agregators-schema-metadata.service" ];
wantedBy = [ "multi-user.target" ];
preStart = ''
cp --no-preserve=mode ${configFile} /run/metadata-agregator/config.yaml
${
if db.passwordFile != null then
''
${pkgs.replace-secret}/bin/replace-secret \
'${dbPasswordPlaceholder}' \
'${db.passwordFile}' \
/run/metadata-agregator/config.yaml
''
else
''
${pkgs.gnused}/bin/sed -i "s/${dbPasswordPlaceholder}//" /run/metadata-agregator/config.yaml
''
}
'';
serviceConfig = {
Type = "simple";
ExecStart = "${lib.getExe metaCfg.package} -config /run/metadata-agregator/config.yaml";
User = cfg.user;
Group = cfg.group;
RuntimeDirectory = "metadata-agregator";
RuntimeDirectoryMode = "0750";
Restart = "on-failure";
RestartSec = 5;
ProtectSystem = "strict";
PrivateTmp = true;
NoNewPrivileges = true;
ProtectHome = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectControlGroups = true;
};
};
networking.firewall.allowedTCPPorts = lib.mkIf metaCfg.openFirewall (
[ metaCfg.port ] ++ lib.optional metaCfg.metrics.enable metaCfg.metrics.port
);
};
}
+275
View File
@@ -0,0 +1,275 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
maCfg = cfg.music-agregator;
pgCfg = cfg.postgres;
qbtCfg = cfg.qbittorrent;
jackettCfg = cfg.jackett;
metaCfg = cfg.metadata-agregator;
mfsCfg = cfg.musicfs;
vpnCfg = cfg.vpn;
yaml = pkgs.formats.yaml { };
vpnEnabled = vpnCfg.enable && qbtCfg.vpn.enable or false;
namespaceAddr = "192.168.15.1";
musicfsEnabled = mfsCfg.enable;
dbPasswordPlaceholder = "@MUSIC_DB_PASSWORD@";
qbtPasswordPlaceholder = "@QBT_PASSWORD@";
# Auto-wire qBittorrent address based on VPN
qbtAddr =
if maCfg.qbittorrentUrl != null then
maCfg.qbittorrentUrl
else if vpnEnabled then
"http://${namespaceAddr}:${toString qbtCfg.webuiPort}"
else
"http://127.0.0.1:${toString qbtCfg.webuiPort}";
dbHost = if pgCfg.host == "/run/postgresql" then "localhost" else pgCfg.host;
configFile = yaml.generate "music-agregator.yaml" {
app = {
host = maCfg.host;
port = toString maCfg.port;
};
database = {
url = "postgresql://${pgCfg.musicDatabase.user}:${dbPasswordPlaceholder}@${dbHost}:${toString pgCfg.port}/${pgCfg.musicDatabase.name}?sslmode=disable";
};
indexer = {
url = maCfg.jackettUrl;
type = "jackett";
api_key = "@JACKETT_API_KEY@";
}
// lib.optionalAttrs (maCfg.indexerCache.enable) {
cache = {
enabled = true;
refresh_interval = maCfg.indexerCache.refreshInterval;
ttl = maCfg.indexerCache.ttl;
};
};
torrent = {
client_type = "qbittorrent";
url = qbtAddr;
username = qbtCfg.username;
password = qbtPasswordPlaceholder;
# No container_name — native paths, no Docker
};
metadata = {
endpoint = maCfg.metadataEndpoint;
};
musicfs = {
enabled = musicfsEnabled;
endpoint = maCfg.musicfsEndpoint;
origin_id = lib.optionalString musicfsEnabled mfsCfg.originId;
origin_root = lib.optionalString musicfsEnabled (toString mfsCfg.originPath);
timeout_seconds = maCfg.musicfsTimeoutSeconds;
};
};
in
{
options.services.agregators.music-agregator = {
enable = lib.mkEnableOption "music-agregator orchestrator service";
package = lib.mkOption {
type = lib.types.package;
description = "The music-agregator package.";
};
host = lib.mkOption {
type = lib.types.str;
default = "0.0.0.0";
description = "gRPC server bind address.";
};
port = lib.mkOption {
type = lib.types.port;
default = 3000;
description = "gRPC server port.";
};
# --- Auto-wired endpoints (override for external services) ---
metadataEndpoint = lib.mkOption {
type = lib.types.str;
default = "localhost:${toString metaCfg.port}";
defaultText = lib.literalExpression ''"localhost:''${toString cfg.metadata-agregator.port}"'';
description = "metadata-agregator gRPC endpoint.";
};
musicfsEndpoint = lib.mkOption {
type = lib.types.str;
default = "localhost:${toString mfsCfg.port}";
defaultText = lib.literalExpression ''"localhost:''${toString cfg.musicfs.port}"'';
description = "musicfs gRPC endpoint.";
};
musicfsTimeoutSeconds = lib.mkOption {
type = lib.types.int;
default = 300;
description = "Timeout in seconds for musicfs operations.";
};
jackettUrl = lib.mkOption {
type = lib.types.str;
default = "http://127.0.0.1:${toString jackettCfg.port}";
defaultText = lib.literalExpression ''"http://127.0.0.1:''${toString cfg.jackett.port}"'';
description = "Jackett HTTP URL.";
};
jackettApiKeyFile = lib.mkOption {
type = lib.types.path;
description = "File containing the Jackett API key.";
};
qbittorrentUrl = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
qBittorrent WebUI URL override.
If null, auto-detected from VPN and port config.
'';
};
qbittorrentPasswordFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = qbtCfg.passwordFile;
defaultText = lib.literalExpression "cfg.qbittorrent.passwordFile";
description = "File containing qBittorrent WebUI password.";
};
indexerCache = {
enable = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Enable Jackett indexer cache with background refresh.";
};
refreshInterval = lib.mkOption {
type = lib.types.str;
default = "30m";
description = "Background refresh interval.";
};
ttl = lib.mkOption {
type = lib.types.str;
default = "1h";
description = "Cache entry time-to-live.";
};
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Open the gRPC port in the firewall.";
};
};
config = lib.mkIf (cfg.enable && maCfg.enable) {
assertions = [
{
assertion = metaCfg.enable;
message = "services.agregators.music-agregator requires services.agregators.metadata-agregator.enable";
}
{
assertion = jackettCfg.enable || maCfg.jackettUrl != "http://127.0.0.1:${toString jackettCfg.port}";
message = "services.agregators.music-agregator requires either jackett enabled or a custom jackettUrl";
}
{
assertion = qbtCfg.enable || maCfg.qbittorrentUrl != null;
message = "services.agregators.music-agregator requires either qbittorrent enabled or a custom qbittorrentUrl";
}
];
systemd.services.music-agregator = {
description = "music-agregator - Music automation orchestrator";
after = [
"network.target"
"agregators-schema-music.service"
"metadata-agregator.service"
]
++ lib.optional jackettCfg.enable "jackett.service"
++ lib.optional qbtCfg.enable "qbittorrent.service"
++ lib.optional musicfsEnabled "musicfs.service";
requires = [
"agregators-schema-music.service"
"metadata-agregator.service"
];
wants =
lib.optional jackettCfg.enable "jackett.service"
++ lib.optional qbtCfg.enable "qbittorrent.service"
++ lib.optional musicfsEnabled "musicfs.service";
wantedBy = [ "multi-user.target" ];
preStart = ''
cp --no-preserve=mode ${configFile} /run/music-agregator/config.yaml
# Inject database password
${
if pgCfg.musicDatabase.passwordFile != null then
''
${pkgs.replace-secret}/bin/replace-secret \
'${dbPasswordPlaceholder}' \
'${pgCfg.musicDatabase.passwordFile}' \
/run/music-agregator/config.yaml
''
else
''
${pkgs.gnused}/bin/sed -i "s/${dbPasswordPlaceholder}//" /run/music-agregator/config.yaml
''
}
# Inject Jackett API key
${pkgs.replace-secret}/bin/replace-secret \
'@JACKETT_API_KEY@' \
'${maCfg.jackettApiKeyFile}' \
/run/music-agregator/config.yaml
# Inject qBittorrent password
${
if maCfg.qbittorrentPasswordFile != null then
''
${pkgs.replace-secret}/bin/replace-secret \
'${qbtPasswordPlaceholder}' \
'${maCfg.qbittorrentPasswordFile}' \
/run/music-agregator/config.yaml
''
else
''
${pkgs.gnused}/bin/sed -i "s/${qbtPasswordPlaceholder}//" /run/music-agregator/config.yaml
''
}
'';
serviceConfig = {
Type = "simple";
ExecStart = "${lib.getExe maCfg.package} -config /run/music-agregator/config.yaml";
User = cfg.user;
Group = cfg.group;
RuntimeDirectory = "music-agregator";
RuntimeDirectoryMode = "0750";
Restart = "on-failure";
RestartSec = 5;
ProtectSystem = "strict";
PrivateTmp = true;
NoNewPrivileges = true;
ProtectHome = true;
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectControlGroups = true;
};
};
networking.firewall.allowedTCPPorts = lib.mkIf maCfg.openFirewall [ maCfg.port ];
};
}
+142
View File
@@ -0,0 +1,142 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
mfsCfg = cfg.musicfs;
toml = pkgs.formats.toml { };
configFile = toml.generate "musicfs.toml" (
{
mount_point = mfsCfg.mountPoint;
cache_dir = mfsCfg.cacheDir;
origins = [
{
id = mfsCfg.originId;
origin_type = "local";
priority = 1;
enabled = true;
path = mfsCfg.originPath;
}
];
}
// lib.optionalAttrs (mfsCfg.cache != { }) { cache = mfsCfg.cache; }
// lib.optionalAttrs (mfsCfg.logging != { }) { logging = mfsCfg.logging; }
// mfsCfg.extraConfig
);
in
{
options.services.agregators.musicfs = {
enable = lib.mkEnableOption "musicfs virtual FUSE filesystem";
package = lib.mkOption {
type = lib.types.package;
description = "The musicfs package.";
};
port = lib.mkOption {
type = lib.types.port;
default = 50052;
description = "gRPC control API port.";
};
mountPoint = lib.mkOption {
type = lib.types.path;
default = "/mnt/music";
description = "Where to mount the virtual filesystem.";
};
cacheDir = lib.mkOption {
type = lib.types.path;
default = "${cfg.stateDir}/musicfs/cache";
defaultText = lib.literalExpression ''"''${cfg.stateDir}/musicfs/cache"'';
description = "Directory for cache data (CAS chunks, metadata, search index).";
};
originPath = lib.mkOption {
type = lib.types.path;
default = "${cfg.mediaDir}/downloads";
defaultText = lib.literalExpression ''"''${cfg.mediaDir}/downloads"'';
description = "Source directory for music files (typically qBittorrent download dir).";
};
originId = lib.mkOption {
type = lib.types.str;
default = "local-storage";
description = "Origin identifier used by musicfs and music-agregator.";
};
cache = lib.mkOption {
type = lib.types.attrsOf toml.type;
default = { };
description = "Cache settings passed to [cache] in config.";
};
logging = lib.mkOption {
type = lib.types.attrsOf toml.type;
default = { };
description = "Logging settings passed to [logging] in config.";
};
extraConfig = lib.mkOption {
type = lib.types.attrsOf toml.type;
default = { };
description = "Additional top-level config keys.";
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Open the gRPC port in the firewall.";
};
};
config = lib.mkIf (cfg.enable && mfsCfg.enable) {
programs.fuse.userAllowOther = true;
systemd.tmpfiles.rules = [
"d '${mfsCfg.mountPoint}' 0755 ${cfg.user} ${cfg.group} - -"
"d '${mfsCfg.cacheDir}' 0750 ${cfg.user} ${cfg.group} - -"
];
systemd.services.musicfs = {
description = "MusicFS - Virtual FUSE Filesystem for Music";
after = [
"network.target"
"local-fs.target"
];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "notify";
ExecStart = "${lib.getExe mfsCfg.package} mount --config ${configFile} --grpc-port ${toString mfsCfg.port}";
ExecStopPost = "${pkgs.fuse3}/bin/fusermount3 -u ${mfsCfg.mountPoint}";
User = cfg.user;
Group = cfg.group;
Restart = "on-failure";
RestartSec = 5;
ProtectSystem = "strict";
ReadWritePaths = [
mfsCfg.mountPoint
mfsCfg.cacheDir
mfsCfg.originPath
];
PrivateTmp = true;
NoNewPrivileges = true;
ProtectHome = "read-only";
ProtectKernelTunables = true;
ProtectKernelModules = true;
ProtectControlGroups = true;
DeviceAllow = [ "/dev/fuse rw" ];
};
};
networking.firewall.allowedTCPPorts = lib.mkIf mfsCfg.openFirewall [ mfsCfg.port ];
};
}
+237
View File
@@ -0,0 +1,237 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
pgCfg = cfg.postgres;
# For local: run as the db user via peer auth (service runs as that user)
# For remote: run as anyone, use password auth via PGPASSWORD
mkPsql =
db:
lib.concatStringsSep " " (
[
"${pkgs.postgresql}/bin/psql"
]
++ lib.optionals (!isLocal) [
"-h ${lib.escapeShellArg pgCfg.host}"
"-p ${toString pgCfg.port}"
"-U ${lib.escapeShellArg db.user}"
]
++ [
"-d ${lib.escapeShellArg db.name}"
]
);
mkPasswordExport =
db:
if db.passwordFile != null then
''export PGPASSWORD="$(cat ${lib.escapeShellArg db.passwordFile})"''
else
"";
isLocal = pgCfg.host == "/run/postgresql" || pgCfg.host == "localhost" || pgCfg.host == "127.0.0.1";
in
{
options.services.agregators.postgres = {
host = lib.mkOption {
type = lib.types.str;
default = "/run/postgresql";
description = ''
PostgreSQL host. Use a path for unix socket (e.g. /run/postgresql)
or a hostname for TCP.
'';
};
port = lib.mkOption {
type = lib.types.port;
default = 5432;
description = "PostgreSQL port (ignored for unix socket connections).";
};
musicDatabase = {
name = lib.mkOption {
type = lib.types.str;
default = "music_agregator";
description = "Database name for music-agregator.";
};
user = lib.mkOption {
type = lib.types.str;
default = "music_agregator";
description = "Database user for music-agregator.";
};
passwordFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "Password file for music-agregator database. Null for peer auth.";
};
};
metadataDatabase = {
name = lib.mkOption {
type = lib.types.str;
default = "metadata_agregator";
description = "Database name for metadata-agregator.";
};
user = lib.mkOption {
type = lib.types.str;
default = "metadata_agregator";
description = "Database user for metadata-agregator.";
};
passwordFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "Password file for metadata-agregator database. Null for peer auth.";
};
};
settings = lib.mkOption {
type = lib.types.attrsOf lib.types.str;
default = { };
description = "Extra PostgreSQL settings merged into services.postgresql.settings.";
example = {
shared_buffers = "256MB";
effective_cache_size = "768MB";
};
};
};
config = lib.mkIf cfg.enable {
# Create system users for peer auth (local postgres only)
users.users = lib.mkIf isLocal {
${pgCfg.musicDatabase.user} = {
isSystemUser = true;
group = cfg.group;
};
${pgCfg.metadataDatabase.user} = {
isSystemUser = true;
group = cfg.group;
};
};
# Enable and configure local PostgreSQL
services.postgresql = lib.mkIf isLocal {
enable = lib.mkDefault true;
settings = {
shared_preload_libraries = "pg_prewarm";
}
// pgCfg.settings;
ensureDatabases = [
pgCfg.musicDatabase.name
pgCfg.metadataDatabase.name
];
ensureUsers = [
{
name = pgCfg.musicDatabase.user;
ensureDBOwnership = true;
}
{
name = pgCfg.metadataDatabase.user;
ensureDBOwnership = true;
}
];
};
# Schema init for metadata_agregator
systemd.services.agregators-schema-metadata = {
description = "Initialize metadata_agregator database schema";
after = [ "network.target" ] ++ lib.optionals isLocal [ "postgresql.service" ];
requires = lib.optionals isLocal [ "postgresql.service" ];
before = [ "metadata-agregator.service" ];
requiredBy = [ "metadata-agregator.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Run as the db user for peer auth on local postgres
User = if isLocal then pgCfg.metadataDatabase.user else cfg.user;
};
script =
let
psql = mkPsql pgCfg.metadataDatabase;
passwordExport = mkPasswordExport pgCfg.metadataDatabase;
in
''
${passwordExport}
MARKER="${cfg.stateDir}/postgres/.schema-metadata-initialized"
# Wait for database
for i in $(seq 1 30); do
if ${psql} -c "SELECT 1" >/dev/null 2>&1; then break; fi
echo "Waiting for metadata database... ($i/30)"
sleep 1
done
if [ -f "$MARKER" ]; then
echo "metadata_agregator schema already initialized"
exit 0
fi
echo "Applying metadata_agregator schema..."
${psql} -f ${./sql/metadata/001_schema.sql}
touch "$MARKER"
echo "metadata_agregator schema initialized"
'';
};
# Schema init for music_agregator
systemd.services.agregators-schema-music = {
description = "Initialize music_agregator database schema";
after = [ "network.target" ] ++ lib.optionals isLocal [ "postgresql.service" ];
requires = lib.optionals isLocal [ "postgresql.service" ];
before = [ "music-agregator.service" ];
requiredBy = [ "music-agregator.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
# Run as the db user for peer auth on local postgres
User = if isLocal then pgCfg.musicDatabase.user else cfg.user;
};
script =
let
psql = mkPsql pgCfg.musicDatabase;
passwordExport = mkPasswordExport pgCfg.musicDatabase;
in
''
${passwordExport}
MARKER="${cfg.stateDir}/postgres/.schema-music-initialized"
# Wait for database
for i in $(seq 1 30); do
if ${psql} -c "SELECT 1" >/dev/null 2>&1; then break; fi
echo "Waiting for music database... ($i/30)"
sleep 1
done
if [ -f "$MARKER" ]; then
echo "music_agregator schema already initialized"
exit 0
fi
echo "Applying music_agregator schema..."
${psql} -f ${./sql/music/001_river.sql}
${psql} -f ${./sql/music/002_schema.sql}
${psql} -f ${./sql/music/003_event_bus.sql}
touch "$MARKER"
echo "music_agregator schema initialized"
'';
};
};
}
+136
View File
@@ -0,0 +1,136 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.agregators;
qbtCfg = cfg.qbittorrent;
vpnCfg = cfg.vpn;
vpnEnabled = vpnCfg.enable && qbtCfg.vpn.enable;
# Inside VPN namespace, services bind to the namespace address
namespaceAddr = "192.168.15.1";
listenAddr = if vpnEnabled then namespaceAddr else "0.0.0.0";
in
{
options.services.agregators.qbittorrent = {
enable = lib.mkEnableOption "qBittorrent torrent client";
vpn = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Confine qBittorrent to the VPN network namespace.";
};
};
webuiPort = lib.mkOption {
type = lib.types.port;
default = 8080;
description = "qBittorrent WebUI port.";
};
peerPort = lib.mkOption {
type = lib.types.port;
default = 6881;
description = "BitTorrent peer port (opened through VPN).";
};
downloadDir = lib.mkOption {
type = lib.types.path;
default = "${cfg.mediaDir}/downloads";
defaultText = lib.literalExpression ''"''${cfg.mediaDir}/downloads"'';
description = "Directory where qBittorrent saves completed downloads.";
};
username = lib.mkOption {
type = lib.types.str;
default = "admin";
description = "qBittorrent WebUI username.";
};
passwordFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = "File containing qBittorrent WebUI password.";
};
stateDir = lib.mkOption {
type = lib.types.path;
default = "${cfg.stateDir}/qbittorrent";
defaultText = lib.literalExpression ''"''${cfg.stateDir}/qbittorrent"'';
description = "qBittorrent state/config directory.";
};
package = lib.mkPackageOption pkgs "qbittorrent-nox" { };
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Open the WebUI port in the firewall.";
};
};
config = lib.mkIf (cfg.enable && qbtCfg.enable) {
assertions = [
{
assertion = qbtCfg.vpn.enable -> vpnCfg.enable;
message = "services.agregators.qbittorrent.vpn.enable requires services.agregators.vpn.enable";
}
];
systemd.tmpfiles.rules = [
"d '${qbtCfg.stateDir}' 0750 ${cfg.user} ${cfg.group} - -"
"d '${qbtCfg.stateDir}/qBittorrent' 0750 ${cfg.user} ${cfg.group} - -"
"d '${qbtCfg.stateDir}/qBittorrent/config' 0750 ${cfg.user} ${cfg.group} - -"
"d '${qbtCfg.downloadDir}' 0775 ${cfg.user} ${cfg.group} - -"
"d '${qbtCfg.downloadDir}/.incomplete' 0775 ${cfg.user} ${cfg.group} - -"
];
systemd.services.qbittorrent = {
description = "qBittorrent-nox BitTorrent client";
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
Type = "simple";
User = cfg.user;
Group = cfg.group;
ExecStart = lib.concatStringsSep " " [
"${lib.getExe qbtCfg.package}"
"--webui-port=${toString qbtCfg.webuiPort}"
"--profile=${qbtCfg.stateDir}"
];
Restart = "on-failure";
RestartSec = 5;
};
};
# VPN confinement
systemd.services.qbittorrent.vpnConfinement = lib.mkIf vpnEnabled {
enable = true;
vpnNamespace = vpnCfg.namespace;
};
vpnNamespaces.${vpnCfg.namespace} = lib.mkIf vpnEnabled {
portMappings = [
{
from = qbtCfg.webuiPort;
to = qbtCfg.webuiPort;
}
];
openVPNPorts = [
{
port = qbtCfg.peerPort;
protocol = "both";
}
];
};
networking.firewall.allowedTCPPorts = lib.mkIf qbtCfg.openFirewall [ qbtCfg.webuiPort ];
};
}
+194
View File
@@ -0,0 +1,194 @@
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE TABLE artists (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
sort_name TEXT,
artist_type TEXT,
country TEXT,
formed_date DATE,
disbanded_date DATE,
description TEXT,
image_url TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE works (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
title TEXT NOT NULL,
work_type TEXT,
language TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE tracks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
work_id UUID REFERENCES works(id),
title TEXT NOT NULL,
duration_ms INT,
isrc TEXT,
explicit BOOLEAN DEFAULT false,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE labels (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
country TEXT,
founded_date DATE,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE albums (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
label_id UUID REFERENCES labels(id),
title TEXT NOT NULL,
album_type TEXT,
secondary_types TEXT[] DEFAULT '{}',
release_date DATE,
upc TEXT,
total_tracks INT,
total_discs INT DEFAULT 1,
cover_url TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE genres (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL UNIQUE,
parent_id UUID REFERENCES genres(id)
);
CREATE TABLE track_artists (
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
role TEXT DEFAULT 'primary',
position INT DEFAULT 0,
PRIMARY KEY (track_id, artist_id, role)
);
CREATE TABLE album_artists (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
role TEXT DEFAULT 'primary',
position INT DEFAULT 0,
PRIMARY KEY (album_id, artist_id, role)
);
CREATE TABLE album_tracks (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
disc_number INT DEFAULT 1,
track_number INT NOT NULL,
PRIMARY KEY (album_id, track_id)
);
CREATE TABLE work_artists (
work_id UUID REFERENCES works(id) ON DELETE CASCADE,
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
role TEXT DEFAULT 'writer',
PRIMARY KEY (work_id, artist_id, role)
);
CREATE TABLE artist_genres (
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
genre_id UUID REFERENCES genres(id) ON DELETE CASCADE,
PRIMARY KEY (artist_id, genre_id)
);
CREATE TABLE album_genres (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
genre_id UUID REFERENCES genres(id) ON DELETE CASCADE,
PRIMARY KEY (album_id, genre_id)
);
CREATE TABLE similar_artists (
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
similar_artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
score REAL DEFAULT 0.5,
PRIMARY KEY (artist_id, similar_artist_id)
);
CREATE TABLE lyrics (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
content TEXT,
synced_content JSONB,
language TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE playlists (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
description TEXT,
is_public BOOLEAN DEFAULT true,
cover_url TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE playlist_tracks (
playlist_id UUID REFERENCES playlists(id) ON DELETE CASCADE,
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
position INT NOT NULL,
added_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (playlist_id, track_id)
);
CREATE TABLE artist_external_ids (
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT,
fetched_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (artist_id, source, source_id)
);
CREATE TABLE album_external_ids (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT,
fetched_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (album_id, source, source_id)
);
CREATE TABLE track_external_ids (
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT,
fetched_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (track_id, source, source_id)
);
CREATE INDEX idx_artists_name ON artists(name);
CREATE INDEX idx_artists_name_trgm ON artists USING gin (name gin_trgm_ops);
CREATE INDEX idx_artists_source ON artists(source, source_id);
CREATE INDEX idx_tracks_isrc ON tracks(isrc) WHERE isrc IS NOT NULL;
CREATE INDEX idx_tracks_source ON tracks(source, source_id);
CREATE INDEX idx_albums_upc ON albums(upc) WHERE upc IS NOT NULL;
CREATE INDEX idx_albums_source ON albums(source, source_id);
CREATE INDEX idx_albums_release_date ON albums(release_date);
CREATE INDEX idx_genres_name ON genres(name);
CREATE INDEX idx_lyrics_track_id ON lyrics(track_id);
CREATE INDEX idx_playlist_tracks_position ON playlist_tracks(playlist_id, position);
+286
View File
@@ -0,0 +1,286 @@
-- Version 1
CREATE TABLE river_migration(
id bigserial PRIMARY KEY,
created_at timestamptz NOT NULL DEFAULT NOW(),
version bigint NOT NULL,
CONSTRAINT version CHECK (version >= 1)
);
CREATE UNIQUE INDEX ON river_migration USING btree(version);
-- Version 2
CREATE TYPE river_job_state AS ENUM(
'available',
'cancelled',
'completed',
'discarded',
'retryable',
'running',
'scheduled'
);
CREATE TABLE river_job(
-- 8 bytes
id bigserial PRIMARY KEY,
-- 8 bytes (4 bytes + 2 bytes + 2 bytes)
--
-- `state` is kept near the top of the table for operator convenience -- when
-- looking at jobs with `SELECT *` it'll appear first after ID. The other two
-- fields aren't as important but are kept adjacent to `state` for alignment
-- to get an 8-byte block.
state river_job_state NOT NULL DEFAULT 'available',
attempt smallint NOT NULL DEFAULT 0,
max_attempts smallint NOT NULL,
-- 8 bytes each (no alignment needed)
attempted_at timestamptz,
created_at timestamptz NOT NULL DEFAULT NOW(),
finalized_at timestamptz,
scheduled_at timestamptz NOT NULL DEFAULT NOW(),
-- 2 bytes (some wasted padding probably)
priority smallint NOT NULL DEFAULT 1,
-- types stored out-of-band
args jsonb,
attempted_by text[],
errors jsonb[],
kind text NOT NULL,
metadata jsonb NOT NULL DEFAULT '{}',
queue text NOT NULL DEFAULT 'default',
tags varchar(255)[],
CONSTRAINT finalized_or_finalized_at_null CHECK ((state IN ('cancelled', 'completed', 'discarded') AND finalized_at IS NOT NULL) OR finalized_at IS NULL),
CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0),
CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4),
CONSTRAINT queue_length CHECK (char_length(queue) > 0 AND char_length(queue) < 128),
CONSTRAINT kind_length CHECK (char_length(kind) > 0 AND char_length(kind) < 128)
);
-- We may want to consider adding another property here after `kind` if it seems
-- like it'd be useful for something.
CREATE INDEX river_job_kind ON river_job USING btree(kind);
CREATE INDEX river_job_state_and_finalized_at_index ON river_job USING btree(state, finalized_at) WHERE finalized_at IS NOT NULL;
CREATE INDEX river_job_prioritized_fetching_index ON river_job USING btree(state, queue, priority, scheduled_at, id);
CREATE INDEX river_job_args_index ON river_job USING GIN(args);
CREATE INDEX river_job_metadata_index ON river_job USING GIN(metadata);
CREATE OR REPLACE FUNCTION river_job_notify()
RETURNS TRIGGER
AS $$
DECLARE
payload json;
BEGIN
IF NEW.state = 'available' THEN
-- Notify will coalesce duplicate notifications within a transaction, so
-- keep these payloads generalized:
payload = json_build_object('queue', NEW.queue);
PERFORM
pg_notify('river_insert', payload::text);
END IF;
RETURN NULL;
END;
$$
LANGUAGE plpgsql;
CREATE TRIGGER river_notify
AFTER INSERT ON river_job
FOR EACH ROW
EXECUTE PROCEDURE river_job_notify();
CREATE UNLOGGED TABLE river_leader(
-- 8 bytes each (no alignment needed)
elected_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL,
-- types stored out-of-band
leader_id text NOT NULL,
name text PRIMARY KEY,
CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128),
CONSTRAINT leader_id_length CHECK (char_length(leader_id) > 0 AND char_length(leader_id) < 128)
);
-- Version 3
ALTER TABLE river_job ALTER COLUMN tags SET DEFAULT '{}';
UPDATE river_job SET tags = '{}' WHERE tags IS NULL;
ALTER TABLE river_job ALTER COLUMN tags SET NOT NULL;
-- Version 4
-- The args column never had a NOT NULL constraint or default value at the
-- database level, though we tried to ensure one at the application level.
ALTER TABLE river_job ALTER COLUMN args SET DEFAULT '{}';
UPDATE river_job SET args = '{}' WHERE args IS NULL;
ALTER TABLE river_job ALTER COLUMN args SET NOT NULL;
ALTER TABLE river_job ALTER COLUMN args DROP DEFAULT;
-- The metadata column never had a NOT NULL constraint or default value at the
-- database level, though we tried to ensure one at the application level.
ALTER TABLE river_job ALTER COLUMN metadata SET DEFAULT '{}';
UPDATE river_job SET metadata = '{}' WHERE metadata IS NULL;
ALTER TABLE river_job ALTER COLUMN metadata SET NOT NULL;
-- The 'pending' job state will be used for upcoming functionality:
ALTER TYPE river_job_state ADD VALUE IF NOT EXISTS 'pending' AFTER 'discarded';
ALTER TABLE river_job DROP CONSTRAINT finalized_or_finalized_at_null;
ALTER TABLE river_job ADD CONSTRAINT finalized_or_finalized_at_null CHECK (
(finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR
(finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded'))
);
DROP TRIGGER river_notify ON river_job;
DROP FUNCTION river_job_notify;
--
-- Create table `river_queue`.
--
CREATE TABLE river_queue (
name text PRIMARY KEY NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
metadata jsonb NOT NULL DEFAULT '{}' ::jsonb,
paused_at timestamptz,
updated_at timestamptz NOT NULL
);
--
-- Alter `river_leader` to add a default value of 'default` to `name`.
--
ALTER TABLE river_leader
ALTER COLUMN name SET DEFAULT 'default',
DROP CONSTRAINT name_length,
ADD CONSTRAINT name_length CHECK (name = 'default');
-- Version 5
--
-- Rebuild the migration table so it's based on `(line, version)`.
--
DO
$body$
BEGIN
-- Tolerate users who may be using their own migration system rather than
-- River's. If they are, they will have skipped version 001 containing
-- `CREATE TABLE river_migration`, so this table won't exist.
IF (SELECT to_regclass('river_migration') IS NOT NULL) THEN
ALTER TABLE river_migration
RENAME TO river_migration_old;
CREATE TABLE river_migration(
line TEXT NOT NULL,
version bigint NOT NULL,
created_at timestamptz NOT NULL DEFAULT NOW(),
CONSTRAINT line_length CHECK (char_length(line) > 0 AND char_length(line) < 128),
CONSTRAINT version_gte_1 CHECK (version >= 1),
PRIMARY KEY (line, version)
);
INSERT INTO river_migration
(created_at, line, version)
SELECT created_at, 'main', version
FROM river_migration_old;
DROP TABLE river_migration_old;
END IF;
END;
$body$
LANGUAGE 'plpgsql';
--
-- Add `river_job.unique_key` and bring up an index on it.
--
-- These statements use `IF NOT EXISTS` to allow users with a `river_job` table
-- of non-trivial size to build the index `CONCURRENTLY` out of band of this
-- migration, then follow by completing the migration.
ALTER TABLE river_job
ADD COLUMN IF NOT EXISTS unique_key bytea;
CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON river_job (kind, unique_key) WHERE unique_key IS NOT NULL;
--
-- Create `river_client` and derivative.
--
-- This feature hasn't quite yet been implemented, but we're taking advantage of
-- the migration to add the schema early so that we can add it later without an
-- additional migration.
--
CREATE UNLOGGED TABLE river_client (
id text PRIMARY KEY NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
metadata jsonb NOT NULL DEFAULT '{}',
paused_at timestamptz,
updated_at timestamptz NOT NULL,
CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128)
);
-- Differs from `river_queue` in that it tracks the queue state for a particular
-- active client.
CREATE UNLOGGED TABLE river_client_queue (
river_client_id text NOT NULL REFERENCES river_client (id) ON DELETE CASCADE,
name text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now(),
max_workers bigint NOT NULL DEFAULT 0,
metadata jsonb NOT NULL DEFAULT '{}',
num_jobs_completed bigint NOT NULL DEFAULT 0,
num_jobs_running bigint NOT NULL DEFAULT 0,
updated_at timestamptz NOT NULL,
PRIMARY KEY (river_client_id, name),
CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128),
CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0),
CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0)
);
-- Version 6
CREATE OR REPLACE FUNCTION river_job_state_in_bitmask(bitmask BIT(8), state river_job_state)
RETURNS boolean
LANGUAGE SQL
IMMUTABLE
AS $$
SELECT CASE state
WHEN 'available' THEN get_bit(bitmask, 7)
WHEN 'cancelled' THEN get_bit(bitmask, 6)
WHEN 'completed' THEN get_bit(bitmask, 5)
WHEN 'discarded' THEN get_bit(bitmask, 4)
WHEN 'pending' THEN get_bit(bitmask, 3)
WHEN 'retryable' THEN get_bit(bitmask, 2)
WHEN 'running' THEN get_bit(bitmask, 1)
WHEN 'scheduled' THEN get_bit(bitmask, 0)
ELSE 0
END = 1;
$$;
--
-- Add `river_job.unique_states` and bring up an index on it.
--
-- This column may exist already if users manually created the column and index
-- as instructed in the changelog so the index could be created `CONCURRENTLY`.
--
ALTER TABLE river_job ADD COLUMN IF NOT EXISTS unique_states BIT(8);
-- This statement uses `IF NOT EXISTS` to allow users with a `river_job` table
-- of non-trivial size to build the index `CONCURRENTLY` out of band of this
-- migration, then follow by completing the migration.
CREATE UNIQUE INDEX IF NOT EXISTS river_job_unique_idx ON river_job (unique_key)
WHERE unique_key IS NOT NULL
AND unique_states IS NOT NULL
AND river_job_state_in_bitmask(unique_states, state);
-- Remove the old unique index. Users who are actively using the unique jobs
-- feature and who wish to avoid deploy downtime may want od drop this in a
-- subsequent migration once all jobs using the old unique system have been
-- completed (i.e. no more rows with non-null unique_key and null
-- unique_states).
DROP INDEX river_job_kind_unique_key_idx;
+166
View File
@@ -0,0 +1,166 @@
CREATE TYPE monitor_state AS ENUM ('monitored', 'unmonitored', 'excluded');
CREATE TABLE artists (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
external_id VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(500) NOT NULL,
artist_type VARCHAR(50),
country VARCHAR(10),
genres TEXT[],
image_url TEXT,
monitor_state monitor_state NOT NULL DEFAULT 'unmonitored',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE albums (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
external_id VARCHAR(255) NOT NULL UNIQUE,
artist_id UUID NOT NULL REFERENCES artists(id) ON DELETE CASCADE,
title VARCHAR(500) NOT NULL,
album_type VARCHAR(50),
release_date DATE,
total_tracks INT,
total_discs INT DEFAULT 1,
label VARCHAR(255),
genres TEXT[],
cover_url TEXT,
monitor_state monitor_state NOT NULL DEFAULT 'unmonitored',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_albums_artist ON albums(artist_id);
CREATE INDEX idx_albums_monitored ON albums(monitor_state) WHERE monitor_state = 'monitored';
CREATE TABLE tracks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
external_id VARCHAR(255) NOT NULL UNIQUE,
album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE,
title VARCHAR(500) NOT NULL,
duration_ms INT,
isrc VARCHAR(20),
disc_number INT DEFAULT 1,
track_number INT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_tracks_album ON tracks(album_id);
CREATE TABLE torrents (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE,
info_hash VARCHAR(40) NOT NULL UNIQUE,
tracker VARCHAR(100) NOT NULL,
title TEXT NOT NULL,
format VARCHAR(20) NOT NULL,
quality VARCHAR(20),
source VARCHAR(20),
bit_depth INT,
sample_rate INT,
seeders INT DEFAULT 0,
peers INT DEFAULT 0,
size BIGINT,
track_count INT,
has_cover_art BOOLEAN DEFAULT FALSE,
has_cue_sheet BOOLEAN DEFAULT FALSE,
has_rip_log BOOLEAN DEFAULT FALSE,
download_link TEXT,
torrent_file BYTEA,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_torrents_album ON torrents(album_id);
CREATE INDEX idx_torrents_album_format ON torrents(album_id, format);
CREATE TYPE download_state AS ENUM (
'pending', 'downloading', 'completed', 'failed', 'seeding', 'paused'
);
CREATE TABLE downloads (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
torrent_id UUID NOT NULL REFERENCES torrents(id) ON DELETE CASCADE,
album_id UUID NOT NULL,
format VARCHAR(20) NOT NULL,
quality VARCHAR(20),
state download_state NOT NULL DEFAULT 'pending',
qbit_hash VARCHAR(64),
save_path TEXT,
error_message TEXT,
queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
started_at TIMESTAMPTZ,
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX idx_unique_active_torrent
ON downloads(torrent_id)
WHERE state NOT IN ('failed');
CREATE UNIQUE INDEX idx_unique_owned_album_quality
ON downloads(album_id, format, quality)
WHERE state IN ('completed', 'seeding');
CREATE INDEX idx_downloads_state ON downloads(state);
CREATE INDEX idx_downloads_album ON downloads(album_id);
CREATE TABLE download_files (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
download_id UUID NOT NULL REFERENCES downloads(id) ON DELETE CASCADE,
track_id UUID REFERENCES tracks(id),
file_path TEXT NOT NULL,
file_size BIGINT NOT NULL,
file_type VARCHAR(20) NOT NULL,
sha256_hash VARCHAR(64),
verified_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_download_files_download ON download_files(download_id);
CREATE INDEX idx_download_files_hash ON download_files(sha256_hash) WHERE sha256_hash IS NOT NULL;
CREATE TABLE album_releases (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE,
download_id UUID NOT NULL REFERENCES downloads(id) ON DELETE CASCADE,
format VARCHAR(20) NOT NULL,
bit_depth INT,
sample_rate INT,
channels INT DEFAULT 2,
is_lossless BOOLEAN NOT NULL DEFAULT FALSE,
source VARCHAR(20),
total_size BIGINT NOT NULL DEFAULT 0,
total_duration_ms INT NOT NULL DEFAULT 0,
track_count INT NOT NULL DEFAULT 0,
has_cover_art BOOLEAN DEFAULT FALSE,
has_cue_sheet BOOLEAN DEFAULT FALSE,
has_rip_log BOOLEAN DEFAULT FALSE,
path TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_album_releases_album ON album_releases(album_id);
CREATE UNIQUE INDEX idx_album_releases_download ON album_releases(download_id);
CREATE TABLE track_releases (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
album_release_id UUID NOT NULL REFERENCES album_releases(id) ON DELETE CASCADE,
track_id UUID REFERENCES tracks(id),
download_file_id UUID REFERENCES download_files(id),
title VARCHAR(500) NOT NULL,
track_number INT NOT NULL DEFAULT 1,
disc_number INT NOT NULL DEFAULT 1,
duration_ms INT,
format VARCHAR(20) NOT NULL,
bit_depth INT,
sample_rate INT,
channels INT DEFAULT 2,
bitrate_kbps INT,
file_size BIGINT NOT NULL DEFAULT 0,
file_path TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_track_releases_album_release ON track_releases(album_release_id);
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE workflow_runs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
album_id UUID NOT NULL REFERENCES albums(id) ON DELETE CASCADE,
quality VARCHAR(20) NOT NULL,
status VARCHAR(20) NOT NULL DEFAULT 'running',
error_message TEXT,
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
running_lock BOOLEAN GENERATED ALWAYS AS (CASE WHEN status = 'running' THEN TRUE ELSE NULL END) STORED,
CONSTRAINT idx_workflow_runs_active UNIQUE (album_id, quality, running_lock)
);
CREATE INDEX idx_workflow_runs_status ON workflow_runs(status);
CREATE TABLE album_events (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
seq BIGSERIAL NOT NULL,
workflow_run_id UUID NOT NULL REFERENCES workflow_runs(id) ON DELETE CASCADE,
album_id UUID NOT NULL,
event_type VARCHAR(20) NOT NULL,
step VARCHAR(50) NOT NULL,
message TEXT NOT NULL,
data_json JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_album_events_workflow ON album_events(workflow_run_id);
CREATE INDEX idx_album_events_album ON album_events(album_id);
CREATE INDEX idx_album_events_seq ON album_events(seq);
ALTER TYPE download_state ADD VALUE IF NOT EXISTS 'cancelled';
+48
View File
@@ -0,0 +1,48 @@
{
config,
lib,
...
}:
let
cfg = config.services.agregators;
vpnCfg = cfg.vpn;
in
{
options.services.agregators.vpn = {
enable = lib.mkEnableOption "WireGuard VPN via VPN-Confinement for torrent traffic";
wgConfigFile = lib.mkOption {
type = lib.types.path;
description = ''
Path to WireGuard configuration file.
Must not be in the Nix store use a runtime path.
'';
example = "/run/secrets/wireguard.conf";
};
accessibleFrom = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [
"192.168.1.0/24"
"192.168.0.0/24"
"127.0.0.1"
];
description = "CIDRs allowed to reach services inside the VPN namespace.";
};
namespace = lib.mkOption {
type = lib.types.str;
default = "wg";
description = "Name of the VPN network namespace.";
};
};
config = lib.mkIf (cfg.enable && vpnCfg.enable) {
vpnNamespaces.${vpnCfg.namespace} = {
enable = true;
wireguardConfigFile = vpnCfg.wgConfigFile;
accessibleFrom = vpnCfg.accessibleFrom;
};
};
}
+104
View File
@@ -0,0 +1,104 @@
{
pkgs,
nixosModules,
}:
pkgs.testers.nixosTest {
name = "agregators-simple-test";
nodes.machine =
{
config,
pkgs,
...
}:
{
imports = [ nixosModules.default ];
networking.firewall.enable = false;
virtualisation.memorySize = 4096;
virtualisation.diskSize = 8192;
services.agregators = {
enable = true;
mediaDir = "/data/music";
# No VPN in test — no WireGuard endpoint to connect to
# qbittorrent.vpn.enable is false by implication
qbittorrent = {
enable = true;
vpn.enable = false;
};
jackett.enable = true;
metadata-agregator = {
enable = true;
package = config.services.agregators.music-agregator.package;
# In real usage this would be the actual metadata-agregator package.
# For the simple test we just check the module evaluates and
# systemd units are created. The service itself won't start
# without the real binary — that's fine, we test structure here.
};
musicfs = {
enable = true;
package = config.services.agregators.music-agregator.package;
};
music-agregator = {
enable = true;
package = pkgs.hello; # placeholder — real package tested separately
jackettApiKeyFile = "/dev/null";
};
};
};
testScript = ''
machine.wait_for_unit("multi-user.target")
# PostgreSQL should be running with both databases
machine.succeed("systemctl is-active postgresql")
machine.succeed("sudo -u postgres psql -lqt | grep -q music_agregator")
machine.succeed("sudo -u postgres psql -lqt | grep -q metadata_agregator")
# Schema init oneshots should have completed
machine.succeed("systemctl is-active agregators-schema-music")
machine.succeed("systemctl is-active agregators-schema-metadata")
# Verify schemas were applied (marker files exist)
machine.succeed("test -f /var/lib/agregators/postgres/.schema-music-initialized")
machine.succeed("test -f /var/lib/agregators/postgres/.schema-metadata-initialized")
# Verify tables exist in music_agregator
machine.succeed("sudo -u music_agregator psql -d music_agregator -c 'SELECT 1 FROM river_job LIMIT 0'")
machine.succeed("sudo -u music_agregator psql -d music_agregator -c 'SELECT 1 FROM artists LIMIT 0'")
machine.succeed("sudo -u music_agregator psql -d music_agregator -c 'SELECT 1 FROM workflow_runs LIMIT 0'")
# Verify tables exist in metadata_agregator
machine.succeed("sudo -u metadata_agregator psql -d metadata_agregator -c 'SELECT 1 FROM artists LIMIT 0'")
machine.succeed("sudo -u metadata_agregator psql -d metadata_agregator -c 'SELECT 1 FROM albums LIMIT 0'")
# Verify extensions
machine.succeed("sudo -u metadata_agregator psql -d metadata_agregator -c \"SELECT 1 FROM pg_extension WHERE extname='pg_trgm'\" | grep -q 1")
# qBittorrent should be running (no VPN in test)
machine.succeed("systemctl is-active qbittorrent")
# Jackett should be running
machine.succeed("systemctl is-active jackett")
# Directories should exist with correct ownership
machine.succeed("test -d /data/music")
machine.succeed("test -d /data/music/downloads")
machine.succeed("test -d /var/lib/agregators")
# Verify the agregators user/group exist
machine.succeed("id agregators")
machine.succeed("getent group agregators")
print("\n=== Agregators Simple Test Completed ===")
'';
}