Optimize GetArtists: parallel artist processing, batch album upserts, batch download lookups, retry on metadata calls

internal/database/album_repository.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 )
 
@@ -60,6 +61,46 @@ func (r *AlbumRepository) Create(ctx context.Context, a *Album) error {
 	return nil
 }
 
+func (r *AlbumRepository) CreateBatch(ctx context.Context, albums []*Album) error {
+	if len(albums) == 0 {
+		return nil
+	}
+
+	batch := &pgx.Batch{}
+	for _, a := range albums {
+		batch.Queue(
+			`INSERT INTO albums (external_id, artist_id, title, album_type, release_date, total_tracks, total_discs, label, genres, cover_url, monitor_state)
+			 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+			 ON CONFLICT (external_id) DO UPDATE SET
+			   title = EXCLUDED.title,
+			   album_type = EXCLUDED.album_type,
+			   release_date = EXCLUDED.release_date,
+			   total_tracks = EXCLUDED.total_tracks,
+			   total_discs = EXCLUDED.total_discs,
+			   label = EXCLUDED.label,
+			   genres = EXCLUDED.genres,
+			   cover_url = EXCLUDED.cover_url,
+			   monitor_state = CASE
+			     WHEN albums.monitor_state = 'excluded' THEN albums.monitor_state
+			     WHEN albums.monitor_state = 'monitored' THEN albums.monitor_state
+			     ELSE EXCLUDED.monitor_state
+			   END,
+			   updated_at = NOW()`,
+			a.ExternalID, a.ArtistID, a.Title, a.AlbumType, a.ReleaseDate, a.TotalTracks, a.TotalDiscs, a.Label, a.Genres, a.CoverURL, a.MonitorState,
+		)
+	}
+
+	results := r.pool.SendBatch(ctx, batch)
+	defer results.Close()
+
+	for range albums {
+		if _, err := results.Exec(); err != nil {
+			return fmt.Errorf("batch creating album: %w", err)
+		}
+	}
+	return nil
+}
+
 func (r *AlbumRepository) GetByExternalID(ctx context.Context, externalID string) (*Album, error) {
 	a := &Album{}
 	err := r.pool.QueryRow(ctx,

internal/database/download_repository.go

@@ -153,6 +153,31 @@ func (r *DownloadRepository) GetByID(ctx context.Context, id string) (*Download,
 	return d, nil
 }
 
+func (r *DownloadRepository) GetLatestByAlbumIDs(ctx context.Context, albumIDs []string) (map[string]*Download, error) {
+	if len(albumIDs) == 0 {
+		return nil, nil
+	}
+
+	rows, err := r.pool.Query(ctx,
+		`SELECT DISTINCT ON (album_id) id, torrent_id, album_id, format, quality, state, qbit_hash, save_path, error_message, queued_at, started_at, completed_at, created_at, updated_at
+		 FROM downloads WHERE album_id = ANY($1) ORDER BY album_id, created_at DESC`, albumIDs,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("batch listing downloads: %w", err)
+	}
+	defer rows.Close()
+
+	result := make(map[string]*Download, len(albumIDs))
+	for rows.Next() {
+		d := &Download{}
+		if err := rows.Scan(&d.ID, &d.TorrentID, &d.AlbumID, &d.Format, &d.Quality, &d.State, &d.QbitHash, &d.SavePath, &d.ErrorMessage, &d.QueuedAt, &d.StartedAt, &d.CompletedAt, &d.CreatedAt, &d.UpdatedAt); err != nil {
+			return nil, fmt.Errorf("scanning download: %w", err)
+		}
+		result[d.AlbumID] = d
+	}
+	return result, nil
+}
+
 func (r *DownloadRepository) HasAlbumInQuality(ctx context.Context, albumID string, format string, quality string) (bool, error) {
 	var exists bool
 	err := r.pool.QueryRow(ctx,