Files
music-agregator/internal/service.go
T
Alexander b448e076bc Leftovers
2026-07-01 20:07:31 +02:00

1455 lines
43 KiB
Go

package internal
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5"
"github.com/riverqueue/river"
"github.com/rs/zerolog/log"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
metadataPb "homelab.lan/music-agregator/gen/metadata/v1"
pb "homelab.lan/music-agregator/gen/music_agregator/v1"
"homelab.lan/music-agregator/internal/analysis"
"homelab.lan/music-agregator/internal/config"
"homelab.lan/music-agregator/internal/database"
"homelab.lan/music-agregator/internal/indexer"
albumMatcher "homelab.lan/music-agregator/internal/matcher"
"homelab.lan/music-agregator/internal/metadata"
"homelab.lan/music-agregator/internal/musicfs"
"homelab.lan/music-agregator/internal/release"
"homelab.lan/music-agregator/internal/torrent"
torrentParser "homelab.lan/music-agregator/internal/tracker"
"homelab.lan/music-agregator/internal/workers"
)
type parsedItem struct {
item *indexer.SearchItemResult
rel *release.Release
torrentData []byte
realSeeders int
}
type MusicAgregatorService struct {
config config.Config
metadata *metadata.MetadataService
indexer indexer.Searcher
torrentClient torrent.TorrentClient
magnetResolver torrentParser.Resolver
riverClient *river.Client[pgx.Tx]
pathMapper *torrent.PathMapper
torrents *database.TorrentRepository
downloads *database.DownloadRepository
artists *database.ArtistRepository
downloadFiles *database.DownloadFileRepository
albumReleases *database.AlbumReleaseRepository
trackReleases *database.TrackReleaseRepository
analyzer *analysis.ReleaseAnalyzer
workflowRuns *database.WorkflowRunRepository
albumEvents *database.AlbumEventRepository
musicfsClient *musicfs.Client
shutdownCtx context.Context
shutdownCancel context.CancelFunc
}
func NewMusicAgregatorService(cfg config.Config, riverClient *river.Client[pgx.Tx], torrentClient torrent.TorrentClient, pathMapper *torrent.PathMapper, db *database.DB, musicfsClient *musicfs.Client) (*MusicAgregatorService, error) {
idx, err := indexer.NewIndexerService(cfg, riverClient, nil)
if err != nil {
log.Err(err).Msg("failed to create IndexerService")
return nil, err
}
metadataClient, _, err := metadata.NewMetadataClient(cfg.Metadata.Endpoint)
if err != nil {
log.Err(err).Msg("failed to create metadata client")
return nil, err
}
magnetResolver, err := torrentParser.NewMagnetResolver(30 * time.Second)
if err != nil {
log.Err(err).Msg("failed to create magnet resolver")
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
return &MusicAgregatorService{
config: cfg,
metadata: metadata.NewMetadataService(metadataClient, db),
indexer: idx,
torrentClient: torrentClient,
magnetResolver: magnetResolver,
riverClient: riverClient,
pathMapper: pathMapper,
torrents: database.NewTorrentRepository(db.Pool),
downloads: database.NewDownloadRepository(db.Pool),
artists: database.NewArtistRepository(db.Pool),
downloadFiles: database.NewDownloadFileRepository(db.Pool),
albumReleases: database.NewAlbumReleaseRepository(db.Pool),
trackReleases: database.NewTrackReleaseRepository(db.Pool),
analyzer: analysis.NewReleaseAnalyzer(db),
workflowRuns: database.NewWorkflowRunRepository(db.Pool),
albumEvents: database.NewAlbumEventRepository(db.Pool),
musicfsClient: musicfsClient,
shutdownCtx: ctx,
shutdownCancel: cancel,
}, nil
}
func NewMusicAgregatorServiceWithDeps(
metadata *metadata.MetadataService,
searcher indexer.Searcher,
torrentClient torrent.TorrentClient,
magnetResolver torrentParser.Resolver,
riverClient *river.Client[pgx.Tx],
pathMapper *torrent.PathMapper,
db *database.DB,
) *MusicAgregatorService {
ctx, cancel := context.WithCancel(context.Background())
return &MusicAgregatorService{
metadata: metadata,
indexer: searcher,
torrentClient: torrentClient,
magnetResolver: magnetResolver,
riverClient: riverClient,
pathMapper: pathMapper,
torrents: database.NewTorrentRepository(db.Pool),
downloads: database.NewDownloadRepository(db.Pool),
artists: database.NewArtistRepository(db.Pool),
downloadFiles: database.NewDownloadFileRepository(db.Pool),
albumReleases: database.NewAlbumReleaseRepository(db.Pool),
trackReleases: database.NewTrackReleaseRepository(db.Pool),
analyzer: analysis.NewReleaseAnalyzer(db),
workflowRuns: database.NewWorkflowRunRepository(db.Pool),
albumEvents: database.NewAlbumEventRepository(db.Pool),
shutdownCtx: ctx,
shutdownCancel: cancel,
}
}
func (s *MusicAgregatorService) Close() {
if s.shutdownCancel != nil {
s.shutdownCancel()
}
if closer, ok := s.magnetResolver.(interface{ Close() }); ok {
closer.Close()
}
}
func (s *MusicAgregatorService) RecoverWorkflows(ctx context.Context) {
stale, err := s.workflowRuns.GetRunning(ctx)
if err != nil {
log.Error().Err(err).Msg("failed to query stale workflow runs for recovery")
return
}
if len(stale) == 0 {
return
}
for _, run := range stale {
downloads, err := s.downloads.GetByAlbumID(ctx, run.AlbumID)
if err != nil {
log.Error().Err(err).Str("workflow_run_id", run.ID).Msg("failed to query downloads for recovery")
s.workflowRuns.SetFailed(ctx, run.ID, "recovery: failed to query downloads")
continue
}
hasActive := false
for _, d := range downloads {
if d.State == "downloading" || d.State == "completed" || d.State == "seeding" {
hasActive = true
break
}
}
if hasActive {
s.workflowRuns.SetCompleted(ctx, run.ID)
log.Info().Str("workflow_run_id", run.ID).Str("album_id", run.AlbumID).Msg("recovered stale workflow as completed")
} else {
s.workflowRuns.SetFailed(ctx, run.ID, "server restarted during workflow")
log.Warn().Str("workflow_run_id", run.ID).Str("album_id", run.AlbumID).Msg("recovered stale workflow as failed")
}
}
}
func (service *MusicAgregatorService) GetArtists(ctx context.Context, _ *pb.GetArtistsRequest) (*pb.GetArtistsResponse, error) {
dbArtists, err := service.artists.GetAll(ctx)
if err != nil {
log.Error().Err(err).Msg("failed to list artists")
return nil, fmt.Errorf("listing artists: %w", err)
}
artists := make([]*pb.ArtistSummary, len(dbArtists))
var mu sync.Mutex
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(5)
for i, a := range dbArtists {
i, a := i, a
g.Go(func() error {
albums, err := service.buildAlbumsForArtist(gCtx, a)
if err != nil {
log.Warn().Err(err).Str("artist", a.Name).Msg("failed to build album details, returning artist without albums")
}
summary := &pb.ArtistSummary{
Id: a.ID,
ExternalId: a.ExternalID,
Name: a.Name,
ArtistType: a.ArtistType,
Country: a.Country,
Genres: a.Genres,
ImageUrl: a.ImageURL,
MonitorState: toProtoMonitorState(a.MonitorState),
Albums: albums,
}
mu.Lock()
artists[i] = summary
mu.Unlock()
return nil
})
}
_ = g.Wait()
return &pb.GetArtistsResponse{Artists: artists}, nil
}
func (service *MusicAgregatorService) buildAlbumsForArtist(ctx context.Context, artist *database.Artist) ([]*pb.AlbumDetail, error) {
var metadataAlbums []*metadataPb.Album
var err error
for attempt := 0; attempt < 3; attempt++ {
metadataAlbums, err = service.metadata.GetArtistAlbums(ctx, artist.ExternalID)
if err == nil {
break
}
log.Warn().Err(err).Int("attempt", attempt+1).Str("artist", artist.Name).Msg("metadata GetArtistAlbums failed, retrying")
time.Sleep(time.Duration(attempt+1) * 200 * time.Millisecond)
}
if err != nil {
return nil, fmt.Errorf("fetching metadata albums: %w", err)
}
service.metadata.PersistAlbumsForArtist(ctx, metadataAlbums, artist.ID, database.Unmonitored)
dbAlbums, err := service.metadata.GetAlbumsByArtistID(ctx, artist.ID)
if err != nil {
log.Warn().Err(err).Str("artist_id", artist.ID).Msg("failed to get local albums")
dbAlbums = nil
}
dbAlbumsByExternalID := make(map[string]*database.Album, len(dbAlbums))
albumIDs := make([]string, 0, len(dbAlbums))
for _, a := range dbAlbums {
dbAlbumsByExternalID[a.ExternalID] = a
albumIDs = append(albumIDs, a.ID)
}
downloadsByAlbumID, _ := service.downloads.GetByAlbumIDs(ctx, albumIDs)
albums := make([]*pb.AlbumDetail, 0, len(metadataAlbums))
for _, ma := range metadataAlbums {
detail := &pb.AlbumDetail{
ExternalId: ma.GetId(),
Title: ma.GetTitle(),
AlbumType: ma.GetAlbumType(),
ReleaseDate: ma.GetReleaseDate(),
TotalTracks: ma.GetTotalTracks(),
TotalDiscs: ma.GetTotalDiscs(),
CoverUrl: ma.GetCoverUrl(),
}
if ma.GetLabel() != nil {
detail.Label = ma.GetLabel().GetName()
}
for _, g := range ma.GetGenres() {
detail.Genres = append(detail.Genres, g.GetName())
}
if dbAlbum, ok := dbAlbumsByExternalID[ma.GetId()]; ok {
detail.Id = dbAlbum.ID
detail.MonitorState = toProtoMonitorState(dbAlbum.MonitorState)
if downloads, ok := downloadsByAlbumID[dbAlbum.ID]; ok {
for _, d := range downloads {
detail.Downloads = append(detail.Downloads, &pb.DownloadInfo{
State: d.State,
Format: d.Format,
Quality: d.Quality,
SavePath: derefStr(d.SavePath),
})
}
}
} else {
detail.MonitorState = pb.MonitorState_MONITOR_STATE_UNMONITORED
}
albums = append(albums, detail)
}
return albums, nil
}
func (service *MusicAgregatorService) GetAlbum(ctx context.Context, req *pb.GetAlbumRequest) (*pb.GetAlbumResponse, error) {
dbAlbum, err := service.metadata.GetAlbumByID(ctx, req.GetAlbumId())
if err != nil {
return nil, fmt.Errorf("album not found: %w", err)
}
info, err := service.buildAlbumInfo(ctx, dbAlbum)
if err != nil {
return nil, err
}
return &pb.GetAlbumResponse{Info: info}, nil
}
func (service *MusicAgregatorService) buildAlbumInfo(ctx context.Context, dbAlbum *database.Album) (*pb.AlbumInfo, error) {
metadataAlbum, err := service.metadata.GetAlbum(ctx, dbAlbum.ExternalID)
if err != nil {
log.Error().Err(err).Str("album_id", dbAlbum.ExternalID).Msg("failed to get album from metadata")
return nil, fmt.Errorf("fetching album: %w", err)
}
metadataTracks, err := service.metadata.GetAlbumTracks(ctx, dbAlbum.ExternalID)
if err != nil {
log.Warn().Err(err).Str("album_id", dbAlbum.ExternalID).Msg("failed to get tracks from metadata")
}
service.metadata.PersistTracks(ctx, dbAlbum.ID, metadataTracks)
album := &pb.AlbumDetail{
Id: dbAlbum.ID,
ExternalId: metadataAlbum.GetId(),
Title: metadataAlbum.GetTitle(),
AlbumType: metadataAlbum.GetAlbumType(),
ReleaseDate: metadataAlbum.GetReleaseDate(),
TotalTracks: metadataAlbum.GetTotalTracks(),
TotalDiscs: metadataAlbum.GetTotalDiscs(),
CoverUrl: metadataAlbum.GetCoverUrl(),
MonitorState: toProtoMonitorState(dbAlbum.MonitorState),
}
if metadataAlbum.GetLabel() != nil {
album.Label = metadataAlbum.GetLabel().GetName()
}
for _, g := range metadataAlbum.GetGenres() {
album.Genres = append(album.Genres, g.GetName())
}
downloads, err := service.downloads.GetByAlbumID(ctx, dbAlbum.ID)
if err == nil && len(downloads) > 0 {
for _, d := range downloads {
album.Downloads = append(album.Downloads, &pb.DownloadInfo{
State: d.State,
Format: d.Format,
Quality: d.Quality,
SavePath: derefStr(d.SavePath),
})
}
}
var bestDownload *database.Download
for _, d := range downloads {
if d.State == "completed" || d.State == "seeding" {
bestDownload = d
break
}
}
if bestDownload == nil && len(downloads) > 0 {
bestDownload = downloads[0]
}
var downloadFilesByTrackID map[string]*database.DownloadFile
if bestDownload != nil {
files, err := service.downloadFiles.GetByDownloadID(ctx, bestDownload.ID)
if err == nil {
downloadFilesByTrackID = make(map[string]*database.DownloadFile, len(files))
for _, f := range files {
if f.TrackID != nil {
downloadFilesByTrackID[*f.TrackID] = f
}
}
}
}
dbTracks, _ := service.metadata.GetTracksByAlbumID(ctx, dbAlbum.ID)
dbTracksByExternalID := make(map[string]*database.Track, len(dbTracks))
for _, t := range dbTracks {
dbTracksByExternalID[t.ExternalID] = t
}
var trackReleasesByTrackID map[string]*database.TrackRelease
albumReleases, err := service.albumReleases.GetByAlbumID(ctx, dbAlbum.ID)
if err == nil && len(albumReleases) > 0 {
ar := albumReleases[0]
album.Release = &pb.AlbumReleaseDetail{
Id: ar.ID,
Format: ar.Format,
Channels: int32(ar.Channels),
IsLossless: ar.IsLossless,
TotalSize: ar.TotalSize,
TotalDurationMs: int32(ar.TotalDurationMs),
TrackCount: int32(ar.TrackCount),
HasCoverArt: ar.HasCoverArt,
HasCueSheet: ar.HasCueSheet,
HasRipLog: ar.HasRipLog,
Path: ar.Path,
}
if ar.BitDepth != nil {
album.Release.BitDepth = int32(*ar.BitDepth)
}
if ar.SampleRate != nil {
album.Release.SampleRate = int32(*ar.SampleRate)
}
if ar.Source != nil {
album.Release.Source = *ar.Source
}
trs, err := service.trackReleases.GetByAlbumReleaseID(ctx, ar.ID)
if err == nil {
trackReleasesByTrackID = make(map[string]*database.TrackRelease, len(trs))
for _, tr := range trs {
if tr.TrackID != nil {
trackReleasesByTrackID[*tr.TrackID] = tr
}
}
}
}
tracks := make([]*pb.TrackDetail, 0, len(metadataTracks))
for _, mt := range metadataTracks {
td := &pb.TrackDetail{
ExternalId: mt.GetId(),
Title: mt.GetTitle(),
DurationMs: mt.GetDurationMs(),
DiscNumber: mt.GetDiscNumber(),
TrackNumber: mt.GetTrackNumber(),
Isrc: mt.GetIsrc(),
Explicit: mt.GetExplicit(),
}
for _, ac := range mt.GetArtists() {
td.Artists = append(td.Artists, &pb.ArtistCredit{
Id: ac.GetArtist().GetId(),
Name: ac.GetArtist().GetName(),
})
}
if dbTrack, ok := dbTracksByExternalID[mt.GetId()]; ok {
td.Id = dbTrack.ID
if tr, ok := trackReleasesByTrackID[dbTrack.ID]; ok {
td.FilePath = tr.FilePath
td.FileSize = tr.FileSize
td.Format = tr.Format
td.Channels = int32(tr.Channels)
if tr.DurationMs != nil {
td.DurationMs = int32(*tr.DurationMs)
}
if tr.BitDepth != nil {
td.BitDepth = int32(*tr.BitDepth)
}
if tr.SampleRate != nil {
td.SampleRate = int32(*tr.SampleRate)
}
if tr.BitrateKbps != nil {
td.BitrateKbps = int32(*tr.BitrateKbps)
}
} else if df, ok := downloadFilesByTrackID[dbTrack.ID]; ok {
td.FilePath = df.FilePath
td.FileSize = df.FileSize
td.Format = df.FileType
}
}
tracks = append(tracks, td)
}
return &pb.AlbumInfo{
Album: album,
Tracks: tracks,
}, nil
}
func (service *MusicAgregatorService) AnalyzeAlbumRelease(ctx context.Context, req *pb.AnalyzeAlbumReleaseRequest) (*pb.AnalyzeAlbumReleaseResponse, error) {
dbAlbum, err := service.metadata.GetAlbumByID(ctx, req.GetAlbumId())
if err != nil {
return nil, fmt.Errorf("album not found: %w", err)
}
downloads, err := service.downloads.GetByAlbumID(ctx, dbAlbum.ID)
if err != nil || len(downloads) == 0 {
return nil, fmt.Errorf("no downloads found for album")
}
var download *database.Download
for _, d := range downloads {
if d.State == "completed" || d.State == "seeding" {
download = d
break
}
}
if download == nil {
return nil, fmt.Errorf("no completed download found for album")
}
contentPath := ""
existingRelease, err := service.albumReleases.GetByDownloadID(ctx, download.ID)
if err == nil {
contentPath = existingRelease.Path
}
if contentPath == "" && download.QbitHash != "" {
torrents, err := service.torrentClient.Find(torrent.FindOptions{Hash: download.QbitHash})
if err == nil && len(torrents) > 0 {
contentPath = torrents[0].ContentPath
if service.pathMapper != nil {
contentPath = service.pathMapper.ToHost(contentPath)
}
}
}
if contentPath == "" {
return nil, fmt.Errorf("cannot determine content path for download")
}
_, _, err = service.analyzer.AnalyzeAndPersist(ctx, download.ID, contentPath)
if err != nil {
return nil, fmt.Errorf("analyzing release: %w", err)
}
info, err := service.buildAlbumInfo(ctx, dbAlbum)
if err != nil {
return nil, err
}
return &pb.AnalyzeAlbumReleaseResponse{Info: info}, nil
}
func (service *MusicAgregatorService) MonitorAlbum(ctx context.Context, req *pb.MonitorAlbumRequest) (*pb.MonitorAlbumResponse, error) {
album, err := service.metadata.GetAlbum(ctx, req.GetAlbumId())
if err != nil {
log.Error().Err(err).Str("album_id", req.GetAlbumId()).Msg("failed to get album")
return nil, err
}
dbAlbum, _ := service.metadata.GetAlbumByExternalID(ctx, album.GetId())
if dbAlbum != nil {
service.metadata.SetAlbumMonitorState(ctx, dbAlbum.ID, database.Monitored)
dbAlbum.MonitorState = database.Monitored
qualityStr := normalizeQuality(req.GetQuality(), 0, 0)
owned, err := service.downloads.HasAlbumInQuality(ctx, dbAlbum.ID, req.GetQuality().String(), qualityStr)
if err == nil && owned {
log.Info().Str("album", dbAlbum.Title).Str("quality", qualityStr).Msg("album already owned in requested quality")
return service.buildMonitorAlbumResponse(ctx, album, dbAlbum, nil), nil
}
}
searchResult, err := service.searchIndexer(album, req.GetIndexerOptions().GetTracker())
if err != nil {
return nil, err
}
parsed := service.parseSearchResults(ctx, searchResult, album, nil)
if len(parsed) == 0 {
return nil, status.Errorf(codes.NotFound, "no torrents found for %s", album.GetTitle())
}
albumArtist := ""
if len(album.GetArtists()) > 0 {
albumArtist = album.GetArtists()[0].GetArtist().GetName()
}
parsed = filterByAlbumMatch(parsed, album.GetTitle(), albumArtist)
if len(parsed) == 0 {
return nil, status.Errorf(codes.NotFound, "no torrents match album title %q", album.GetTitle())
}
filtered := filterByQuality(parsed, req.GetQuality())
if len(filtered) == 0 {
return nil, status.Errorf(codes.NotFound, "no releases match quality filter for %s", album.GetTitle())
}
filtered = filterByActiveSeeders(filtered)
if len(filtered) == 0 {
return nil, status.Errorf(codes.NotFound, "no releases with active seeders for %s", album.GetTitle())
}
best := selectBestRelease(filtered, album.GetTitle())
if err := service.addToTorrentClient(best); err != nil {
return nil, err
}
dbAlbum, _ = service.metadata.GetAlbumByExternalID(ctx, album.GetId())
if dbAlbum == nil {
service.metadata.PersistArtist(ctx, album, database.Monitored)
service.metadata.PersistAlbum(ctx, album, database.Monitored)
dbAlbum, _ = service.metadata.GetAlbumByExternalID(ctx, album.GetId())
}
if dbAlbum != nil {
service.saveTorrentAndDownload(ctx, dbAlbum.ID, "", best)
} else {
log.Warn().Str("album_id", req.GetAlbumId()).Msg("album not in DB after persist attempt, skipping torrent/download persistence")
}
return service.buildMonitorAlbumResponse(ctx, album, dbAlbum, &best), nil
}
func (service *MusicAgregatorService) searchIndexer(album *metadataPb.Album, tracker string) (*indexer.SearchResponse, error) {
artistName := ""
if len(album.GetArtists()) > 0 {
artistName = album.GetArtists()[0].GetArtist().GetName()
}
if tracker == "" {
tracker = "all"
}
title := album.GetTitle()
queries := buildSearchQueries(artistName, title)
for _, query := range queries {
result, err := service.indexer.Search(query, -1, tracker)
if err != nil {
log.Error().Err(err).Str("query", query).Msg("indexer search failed")
return nil, err
}
log.Debug().Int("results", len(result.Items)).Str("query", query).Msg("indexer search completed")
if len(result.Items) > 0 {
return result, nil
}
log.Debug().Str("query", query).Msg("no results, trying broader search")
}
return &indexer.SearchResponse{}, nil
}
func buildSearchQueries(artist, title string) []string {
artist = normalizeSearchQuery(artist)
title = normalizeSearchQuery(title)
clean := cleanAlbumTitle(title)
var queries []string
seen := make(map[string]bool)
add := func(q string) {
q = strings.TrimSpace(q)
if q != "" && !seen[strings.ToLower(q)] {
seen[strings.ToLower(q)] = true
queries = append(queries, q)
}
}
withArtist := func(t string) string {
if artist != "" {
return artist + " " + t
}
return t
}
add(withArtist(clean))
noPunct := stripPunctuation(clean)
add(withArtist(noPunct))
if artist != "" {
add(artist)
}
return queries
}
var unicodeReplacer = strings.NewReplacer(
"\u2019", "'", // right single quote
"\u2018", "'", // left single quote
"\u201C", "\"", // left double quote
"\u201D", "\"", // right double quote
"\u2013", "-", // en dash
"\u2014", "-", // em dash
"\u2026", "...", // ellipsis
)
func normalizeSearchQuery(s string) string {
s = unicodeReplacer.Replace(s)
for strings.Contains(s, " ") {
s = strings.ReplaceAll(s, " ", " ")
}
return strings.TrimSpace(s)
}
var editionTagPattern = regexp.MustCompile(`(?i)\b(remaster(?:ed)?|deluxe(?:\s+edition)?|expanded(?:\s+edition)?|anniversary(?:\s+edition)?|special\s+edition|bonus\s+track|limited\s+edition|collector'?s?\s+edition|super\s+deluxe|standard\s+edition)\b`)
var bracketPattern = regexp.MustCompile(`[\(\[\{][^)\]\}]*[\)\]\}]`)
func cleanAlbumTitle(title string) string {
clean := bracketPattern.ReplaceAllString(title, "")
clean = editionTagPattern.ReplaceAllString(clean, "")
clean = strings.TrimRight(clean, " -–—:")
for strings.Contains(clean, " ") {
clean = strings.ReplaceAll(clean, " ", " ")
}
clean = strings.TrimSpace(clean)
if clean == "" {
return title
}
return clean
}
func extractEditionTags(title string) []string {
matches := editionTagPattern.FindAllString(title, -1)
tags := make([]string, len(matches))
for i, m := range matches {
tags[i] = strings.ToLower(strings.TrimSpace(m))
}
return tags
}
func stripPunctuation(s string) string {
var b strings.Builder
for _, r := range s {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == ' ':
b.WriteRune(r)
default:
b.WriteRune(' ')
}
}
result := b.String()
for strings.Contains(result, " ") {
result = strings.ReplaceAll(result, " ", " ")
}
return strings.TrimSpace(result)
}
type resolveProgressFunc func(phase string, current, total int)
const maxMagnetResolves = 5
func (service *MusicAgregatorService) parseSearchResults(ctx context.Context, searchResult *indexer.SearchResponse, album *metadataPb.Album, onProgress resolveProgressFunc) []parsedItem {
var candidates []*indexer.SearchItemResult
for i := range searchResult.Items {
item := searchResult.Items[i]
if item.DownloadLink == "" {
log.Trace().Str("title", item.Title).Msg("skipping item without download link")
continue
}
if item.Seeders == 0 {
log.Warn().Str("title", item.Title).Str("tracker", item.Tracker).Msg("skipping torrent with no seeders")
continue
}
candidates = append(candidates, item)
}
total := len(candidates)
if total == 0 {
return nil
}
progress := func(phase string, current, t int) {
if onProgress != nil {
onProgress(phase, current, t)
}
}
progress("Pre-parsing titles", 0, total)
parser := torrentParser.NewGenericParser()
parsed := make([]parsedItem, total)
for i, item := range candidates {
rel := parser.Parse(item.Title)
parsed[i] = parsedItem{
item: item,
rel: rel,
realSeeders: -1,
}
}
log.Info().Int("candidates", total).Msg("phase 1: title pre-parse complete")
progress("Pre-parsed titles", total, total)
progress("Checking cache", 0, total)
var cacheHits int
for i := range parsed {
p := &parsed[i]
infoHash := torrentParser.ExtractInfoHash(p.item.DownloadLink)
if infoHash == "" {
if h, ok := p.item.Attrs["infohash"]; ok {
infoHash = strings.ToLower(h)
}
}
if infoHash == "" {
continue
}
cached, err := service.torrents.GetByInfoHash(ctx, infoHash)
if err != nil || len(cached.TorrentFile) == 0 {
continue
}
fullParser := torrentParser.NewGenericParser()
p.rel = fullParser.ParseTorrent(cached.TorrentFile, album)
p.torrentData = cached.TorrentFile
cacheHits++
log.Debug().
Str("title", p.item.Title).
Str("hash", infoHash).
Str("format", p.rel.Format.String()).
Msg("cache hit")
}
log.Info().Int("cache_hits", cacheHits).Int("candidates", total).Msg("phase 2: cache lookup complete")
progress("Cache hits", cacheHits, total)
var needsResolve []int
for i := range parsed {
if parsed[i].torrentData == nil {
needsResolve = append(needsResolve, i)
}
}
sort.Slice(needsResolve, func(a, b int) bool {
return parsed[needsResolve[a]].item.Seeders > parsed[needsResolve[b]].item.Seeders
})
if len(needsResolve) > maxMagnetResolves {
needsResolve = needsResolve[:maxMagnetResolves]
}
resolveCount := len(needsResolve)
if resolveCount > 0 {
log.Info().Int("to_resolve", resolveCount).Int("skipped", total-cacheHits-resolveCount).Msg("phase 3: resolving top candidates")
progress("Resolving torrents", 0, resolveCount)
const maxWorkers = 10
sem := make(chan struct{}, maxWorkers)
var wg sync.WaitGroup
var resolved atomic.Int32
for _, idx := range needsResolve {
wg.Add(1)
go func(i int) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
p := &parsed[i]
resolveParser := torrentParser.NewGenericParser()
out := service.resolveRelease(resolveParser, p.item, album)
p.rel = out.rel
p.torrentData = out.torrentData
p.realSeeders = out.realSeeders
n := int(resolved.Add(1))
progress("Resolving torrents", n, resolveCount)
log.Debug().
Str("title", p.item.Title).
Str("format", out.rel.Format.String()).
Int("real_seeders", out.realSeeders).
Msg("resolved")
}(idx)
}
wg.Wait()
}
log.Debug().
Int("total", total).
Int("cache_hits", cacheHits).
Int("resolved", resolveCount).
Int("title_only", total-cacheHits-resolveCount).
Msg("parsing complete")
return parsed
}
type resolveOutput struct {
rel *release.Release
torrentData []byte
realSeeders int
}
func (service *MusicAgregatorService) resolveRelease(parser *torrentParser.GenericParser, item *indexer.SearchItemResult, album *metadataPb.Album) resolveOutput {
if strings.HasPrefix(item.DownloadLink, "magnet:") {
log.Trace().Str("title", item.Title).Int("reported_seeders", item.Seeders).Msg("resolving magnet")
result, err := service.magnetResolver.Resolve(item.DownloadLink)
if err != nil {
log.Warn().Err(err).Str("title", item.Title).Int("reported_seeders", item.Seeders).Msg("magnet resolve failed, falling back to title parse")
return resolveOutput{rel: parser.Parse(item.Title)}
}
log.Debug().
Str("title", item.Title).
Int("reported_seeders", item.Seeders).
Int("real_seeders", result.ConnectedSeeders).
Msg("magnet resolved with seeder info")
return resolveOutput{
rel: parser.ParseTorrent(result.Data, album),
torrentData: result.Data,
realSeeders: result.ConnectedSeeders,
}
}
torrentData, err := downloadTorrentData(item.DownloadLink)
if err != nil {
log.Warn().Err(err).Str("title", item.Title).Msg("failed to download torrent, falling back to title parse")
return resolveOutput{rel: parser.Parse(item.Title)}
}
return resolveOutput{
rel: parser.ParseTorrent(torrentData, album),
torrentData: torrentData,
realSeeders: -1,
}
}
func filterByAlbumMatch(items []parsedItem, albumTitle, artistName string) []parsedItem {
var matched []parsedItem
for _, p := range items {
score := albumMatcher.AlbumMatchScore(albumTitle, p.item.Title, artistName)
if score >= albumMatcher.MatchThreshold {
matched = append(matched, p)
} else {
log.Debug().
Str("title", p.item.Title).
Float64("score", score).
Str("wanted", albumTitle).
Msg("filtered out by album match")
}
}
if len(matched) == 0 {
log.Warn().Str("album", albumTitle).Int("total", len(items)).Msg("no torrents match album title")
}
return matched
}
func filterByQuality(items []parsedItem, quality pb.QualityType) []parsedItem {
var filtered []parsedItem
for _, p := range items {
match := quality == pb.QualityType_QUALITY_UNSPECIFIED ||
(quality == pb.QualityType_QUALITY_LOSSLESS && p.rel.Format.IsLossless()) ||
(quality == pb.QualityType_QUALITY_LOSSY && !p.rel.Format.IsLossless())
if !match {
log.Debug().Str("title", p.item.Title).Str("format", p.rel.Format.String()).Str("wanted", quality.String()).Msg("filtered out by quality")
continue
}
filtered = append(filtered, p)
}
return filtered
}
func filterByActiveSeeders(items []parsedItem) []parsedItem {
var withSeeders []parsedItem
for _, p := range items {
if p.realSeeders > 0 {
withSeeders = append(withSeeders, p)
} else if p.realSeeders == -1 {
withSeeders = append(withSeeders, p)
} else {
log.Debug().
Str("title", p.item.Title).
Int("reported_seeders", p.item.Seeders).
Int("real_seeders", p.realSeeders).
Msg("filtered out: magnet resolved but no active seeders")
}
}
if len(withSeeders) == 0 {
log.Warn().Int("total", len(items)).Msg("no torrents with active seeders, keeping all as fallback")
return items
}
return withSeeders
}
func selectBestRelease(items []parsedItem, albumTitle string) parsedItem {
editionTags := extractEditionTags(albumTitle)
best := items[0]
for _, p := range items[1:] {
if betterRelease(p, best, editionTags) {
best = p
}
}
log.Info().
Str("title", best.item.Title).
Str("format", best.rel.Format.String()).
Int("seeders", best.item.Seeders).
Int("real_seeders", best.realSeeders).
Str("tracker", best.item.Tracker).
Str("hash", best.rel.InfoHash).
Strs("edition_tags", editionTags).
Msg("best release selected")
return best
}
func betterRelease(candidate, current parsedItem, editionTags []string) bool {
cEdition := editionMatchScore(candidate.item.Title, editionTags)
bEdition := editionMatchScore(current.item.Title, editionTags)
if cEdition != bEdition {
return cEdition > bEdition
}
cHasReal := candidate.realSeeders > 0
bHasReal := current.realSeeders > 0
if cHasReal && !bHasReal {
return true
}
if !cHasReal && bHasReal {
return false
}
if cHasReal && bHasReal {
return candidate.realSeeders > current.realSeeders
}
return candidate.item.Seeders > current.item.Seeders
}
func editionMatchScore(torrentTitle string, editionTags []string) int {
if len(editionTags) == 0 {
return 0
}
lower := strings.ToLower(torrentTitle)
score := 0
for _, tag := range editionTags {
if strings.Contains(lower, tag) {
score++
}
}
return score
}
func (service *MusicAgregatorService) addToTorrentClient(best parsedItem) error {
if best.rel.InfoHash != "" {
existing, err := service.torrentClient.Find(torrent.FindOptions{Hash: best.rel.InfoHash})
if err == nil && len(existing) > 0 {
log.Info().Str("hash", best.rel.InfoHash).Str("state", existing[0].State).Msg("torrent already exists in client")
return nil
}
}
savePath := ""
if service.pathMapper != nil {
savePath = service.pathMapper.ContainerDownloadPath()
}
if strings.HasPrefix(best.item.DownloadLink, "magnet:") {
if err := service.torrentClient.AddMagnet(best.item.DownloadLink, savePath); err != nil {
log.Error().Err(err).Str("title", best.item.Title).Msg("failed to add magnet to client")
return err
}
} else {
if len(best.torrentData) == 0 {
log.Error().Str("title", best.item.Title).Msg("no torrent data available")
return fmt.Errorf("no torrent data available for best release")
}
if err := service.torrentClient.AddTorrent(torrent.TorrentFile{
Filename: best.rel.Album + ".torrent",
Data: best.torrentData,
}, savePath); err != nil {
log.Error().Err(err).Str("title", best.item.Title).Msg("failed to add torrent to client")
return err
}
}
log.Info().Str("title", best.item.Title).Str("hash", best.rel.InfoHash).Msg("torrent added to client")
return nil
}
func (service *MusicAgregatorService) saveTorrentAndDownload(ctx context.Context, dbAlbumID string, workflowRunID string, best parsedItem) {
quality := normalizeQuality(pb.QualityType_QUALITY_UNSPECIFIED, best.rel.BitDepth, best.rel.SampleRate)
dbTorrent := &database.Torrent{
AlbumID: dbAlbumID,
InfoHash: best.rel.InfoHash,
Tracker: best.item.Tracker,
Title: best.item.Title,
Format: best.rel.Format.String(),
Quality: quality,
Source: best.rel.Source.String(),
BitDepth: best.rel.BitDepth,
SampleRate: best.rel.SampleRate,
Seeders: best.item.Seeders,
Peers: best.item.Peers,
Size: best.rel.TotalAudioSize,
TrackCount: best.rel.TrackCount,
HasCoverArt: best.rel.HasCoverArt,
HasCueSheet: best.rel.HasCueSheet,
HasRipLog: best.rel.HasRipLog,
DownloadLink: best.item.DownloadLink,
TorrentFile: best.torrentData,
}
if err := service.torrents.Create(ctx, dbTorrent); err != nil {
log.Error().Err(err).Str("hash", best.rel.InfoHash).Msg("failed to save torrent to DB")
return
}
savedTorrent, err := service.torrents.GetByInfoHash(ctx, best.rel.InfoHash)
if err != nil {
log.Error().Err(err).Msg("failed to retrieve saved torrent")
return
}
existingDownload, err := service.downloads.GetActiveByTorrentID(ctx, savedTorrent.ID)
if err == nil && existingDownload != nil {
log.Info().Str("hash", best.rel.InfoHash).Str("state", existingDownload.State).Msg("active download already exists, skipping")
return
}
download := &database.Download{
TorrentID: savedTorrent.ID,
AlbumID: dbAlbumID,
Format: best.rel.Format.String(),
Quality: quality,
State: "downloading",
QbitHash: best.rel.InfoHash,
}
if err := service.downloads.Create(ctx, download); err != nil {
log.Error().Err(err).Msg("failed to save download to DB")
return
}
if service.riverClient != nil {
_, err := service.riverClient.Insert(ctx, workers.PollDownloadArgs{
DownloadID: download.ID,
TorrentHash: best.rel.InfoHash,
CheckInterval: 30 * time.Second,
AlbumID: dbAlbumID,
Quality: quality,
WorkflowRunID: workflowRunID,
}, &river.InsertOpts{
ScheduledAt: time.Now().Add(30 * time.Second),
})
if err != nil {
log.Error().Err(err).Msg("failed to schedule download poll job")
} else {
log.Debug().Str("download_id", download.ID).Str("hash", best.rel.InfoHash).Msg("download poll job scheduled")
}
}
log.Info().Str("hash", best.rel.InfoHash).Str("download_id", download.ID).Msg("torrent and download saved to DB")
}
func normalizeQuality(quality pb.QualityType, bitDepth int, sampleRate int) string {
if bitDepth > 0 && sampleRate > 0 {
return fmt.Sprintf("%d-%d", bitDepth, sampleRate/1000)
}
switch quality {
case pb.QualityType_QUALITY_LOSSLESS:
return "16-44"
case pb.QualityType_QUALITY_LOSSY:
return "320"
default:
return ""
}
}
func buildMonitoredRelease(p parsedItem) *pb.MonitoredRelease {
return &pb.MonitoredRelease{
InfoHash: p.rel.InfoHash,
Artist: p.rel.Artist,
Album: p.rel.Album,
Year: int32(p.rel.Year),
Format: p.rel.Format.String(),
Lossless: p.rel.Format.IsLossless(),
BitDepth: int32(p.rel.BitDepth),
SampleRate: int32(p.rel.SampleRate),
Source: p.rel.Source.String(),
TrackCount: int32(p.rel.TrackCount),
TrackNames: p.rel.TrackNames,
HasCoverArt: p.rel.HasCoverArt,
HasCueSheet: p.rel.HasCueSheet,
HasRipLog: p.rel.HasRipLog,
TotalAudioSize: p.rel.TotalAudioSize,
DownloadLink: p.item.DownloadLink,
Seeders: int32(p.item.Seeders),
Tracker: p.item.Tracker,
}
}
func (service *MusicAgregatorService) buildMonitorAlbumResponse(ctx context.Context, metadataAlbum *metadataPb.Album, dbAlbum *database.Album, best *parsedItem) *pb.MonitorAlbumResponse {
resp := &pb.MonitorAlbumResponse{}
if best != nil {
resp.Release = buildMonitoredRelease(*best)
}
if dbAlbum != nil {
resp.Album = service.buildAlbumDetail(ctx, dbAlbum)
}
if len(metadataAlbum.GetArtists()) > 0 {
dbArtist, err := service.metadata.GetArtistByExternalID(ctx, metadataAlbum.GetArtists()[0].GetArtist().GetId())
if err == nil {
resp.Artist = &pb.ArtistSummary{
Id: dbArtist.ID,
ExternalId: dbArtist.ExternalID,
Name: dbArtist.Name,
ArtistType: dbArtist.ArtistType,
Country: dbArtist.Country,
Genres: dbArtist.Genres,
ImageUrl: dbArtist.ImageURL,
MonitorState: toProtoMonitorState(dbArtist.MonitorState),
}
}
}
return resp
}
func (service *MusicAgregatorService) buildAlbumDetail(ctx context.Context, dbAlbum *database.Album) *pb.AlbumDetail {
detail := &pb.AlbumDetail{
Id: dbAlbum.ID,
ExternalId: dbAlbum.ExternalID,
Title: dbAlbum.Title,
AlbumType: dbAlbum.AlbumType,
TotalTracks: int32(dbAlbum.TotalTracks),
TotalDiscs: int32(dbAlbum.TotalDiscs),
Label: dbAlbum.Label,
Genres: dbAlbum.Genres,
CoverUrl: dbAlbum.CoverURL,
MonitorState: toProtoMonitorState(dbAlbum.MonitorState),
}
if dbAlbum.ReleaseDate != nil {
detail.ReleaseDate = dbAlbum.ReleaseDate.Format("2006-01-02")
}
downloads, err := service.downloads.GetByAlbumID(ctx, dbAlbum.ID)
if err == nil {
for _, d := range downloads {
detail.Downloads = append(detail.Downloads, &pb.DownloadInfo{
State: d.State,
Format: d.Format,
Quality: d.Quality,
SavePath: derefStr(d.SavePath),
})
}
}
return detail
}
func toProtoMonitorState(state database.MonitorState) pb.MonitorState {
switch state {
case database.Monitored:
return pb.MonitorState_MONITOR_STATE_MONITORED
case database.Unmonitored:
return pb.MonitorState_MONITOR_STATE_UNMONITORED
case database.Excluded:
return pb.MonitorState_MONITOR_STATE_EXCLUDED
default:
return pb.MonitorState_MONITOR_STATE_UNSPECIFIED
}
}
func derefStr(s *string) string {
if s == nil {
return ""
}
return *s
}
func downloadTorrentData(url string) ([]byte, error) {
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("downloading torrent: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("torrent download returned HTTP %d", resp.StatusCode)
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading torrent data: %w", err)
}
return data, nil
}
func (service *MusicAgregatorService) ResyncAlbum(ctx context.Context, albumID string) (*pb.ResyncAlbumResponse, error) {
if service.musicfsClient == nil {
return nil, status.Error(codes.FailedPrecondition, "musicfs not configured")
}
download, err := service.downloads.GetLatestCompleted(ctx, albumID)
if err != nil {
return nil, status.Errorf(codes.NotFound, "no completed download for album %s: %v", albumID, err)
}
subdir := ""
if download.QbitHash != "" {
results, err := service.torrentClient.Find(torrent.FindOptions{Hash: download.QbitHash})
if err == nil && len(results) > 0 {
contentPath := results[0].ContentPath
if service.pathMapper != nil {
contentPath = service.pathMapper.ToHost(contentPath)
}
subdir = musicfs.DeriveSubdir(service.config.MusicFS.OriginRoot, contentPath)
}
}
log.Info().
Str("album_id", albumID).
Str("subdir", subdir).
Msg("resyncing album with musicfs")
result, err := service.musicfsClient.TriggerRescan(ctx, service.config.MusicFS.OriginID, subdir)
if err != nil {
return nil, status.Errorf(codes.Internal, "musicfs rescan failed: %v", err)
}
log.Info().
Int("new_files", len(result.NewFiles)).
Uint64("bytes_synced", result.BytesSynced).
Msg("musicfs rescan complete")
var filesEnriched int
if len(result.NewFiles) > 0 {
dbAlbum, err := service.metadata.GetAlbumByID(ctx, albumID)
if err != nil {
log.Warn().Err(err).Str("album_id", albumID).Msg("failed to get album for enrichment")
} else {
albumMeta, err := service.metadata.GetAlbum(ctx, dbAlbum.ExternalID)
if err != nil {
log.Warn().Err(err).Str("external_id", dbAlbum.ExternalID).Msg("failed to get album metadata for enrichment")
} else {
err = service.musicfsClient.EnrichFiles(ctx, result.NewFiles, albumMeta)
if err != nil {
log.Warn().Err(err).Msg("musicfs enrichment failed")
} else {
filesEnriched = len(result.NewFiles)
}
}
}
}
return &pb.ResyncAlbumResponse{
FilesSynced: int32(len(result.NewFiles)),
FilesEnriched: int32(filesEnriched),
BytesSynced: result.BytesSynced,
}, nil
}
func (service *MusicAgregatorService) SearchArtists(ctx context.Context, req *pb.SearchArtistsRequest) (*pb.SearchArtistsResponse, error) {
resp, err := service.metadata.SearchArtists(ctx, req.GetQuery(), req.GetLimit(), req.GetOffset())
if err != nil {
return nil, err
}
artists := make([]*pb.SearchArtistResult, len(resp.GetArtists()))
for i, a := range resp.GetArtists() {
genres := make([]string, len(a.GetGenres()))
for j, g := range a.GetGenres() {
genres[j] = g.GetName()
}
extIDs := make([]*pb.ExternalId, len(a.GetExternalIds()))
for j, e := range a.GetExternalIds() {
extIDs[j] = &pb.ExternalId{
Provider: e.GetSource(),
Id: e.GetSourceId(),
}
}
artists[i] = &pb.SearchArtistResult{
Id: a.GetId(),
Name: a.GetName(),
SortName: a.GetSortName(),
ArtistType: a.GetArtistType(),
Country: a.GetCountry(),
FormedDate: a.GetFormedDate(),
DisbandedDate: a.GetDisbandedDate(),
Description: a.GetDescription(),
ImageUrl: a.GetImageUrl(),
Genres: genres,
ExternalIds: extIDs,
}
}
return &pb.SearchArtistsResponse{
Artists: artists,
Total: resp.GetTotal(),
}, nil
}
func (service *MusicAgregatorService) GetArtistAlbums(ctx context.Context, req *pb.GetArtistAlbumsRequest) (*pb.GetArtistAlbumsResponse, error) {
resp, err := service.metadata.GetArtistAlbumsWithPagination(ctx, req.GetArtistId(), req.GetLimit(), req.GetOffset())
if err != nil {
return nil, err
}
albums := make([]*pb.AlbumResult, len(resp.GetAlbums()))
for i, a := range resp.GetAlbums() {
genres := make([]string, len(a.GetGenres()))
for j, g := range a.GetGenres() {
genres[j] = g.GetName()
}
extIDs := make([]*pb.ExternalId, len(a.GetExternalIds()))
for j, e := range a.GetExternalIds() {
extIDs[j] = &pb.ExternalId{
Provider: e.GetSource(),
Id: e.GetSourceId(),
}
}
artists := make([]*pb.AlbumArtistCredit, len(a.GetArtists()))
for j, ac := range a.GetArtists() {
artists[j] = &pb.AlbumArtistCredit{
Id: ac.GetArtist().GetId(),
Name: ac.GetArtist().GetName(),
Role: ac.GetRole(),
}
}
var label *pb.AlbumLabel
if a.GetLabel() != nil {
label = &pb.AlbumLabel{
Id: a.GetLabel().GetId(),
Name: a.GetLabel().GetName(),
Country: a.GetLabel().GetCountry(),
}
}
albums[i] = &pb.AlbumResult{
Id: a.GetId(),
Title: a.GetTitle(),
AlbumType: a.GetAlbumType(),
ReleaseDate: a.GetReleaseDate(),
Upc: a.GetUpc(),
TotalTracks: a.GetTotalTracks(),
TotalDiscs: a.GetTotalDiscs(),
CoverUrl: a.GetCoverUrl(),
Artists: artists,
Label: label,
Genres: genres,
ExternalIds: extIDs,
}
}
return &pb.GetArtistAlbumsResponse{
Albums: albums,
Total: resp.GetTotal(),
}, nil
}