Optimize GetArtists: parallel artist processing, batch album upserts, batch download lookups, retry on metadata calls

internal/service.go

@@ -6,11 +6,13 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/jackc/pgx/v5"
 	"github.com/riverqueue/river"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/sync/errgroup"
 
 	metadataPb "homelab.lan/music-agregator/gen/metadata/v1"
 	pb "homelab.lan/music-agregator/gen/music_agregator/v1"
@@ -125,38 +127,59 @@ func (service *MusicAgregatorService) GetArtists(ctx context.Context, _ *pb.GetA
 		return nil, fmt.Errorf("listing artists: %w", err)
 	}
 
-	artists := make([]*pb.ArtistSummary, 0, len(dbArtists))
-	for _, a := range dbArtists {
-		albums, err := service.buildAlbumsForArtist(ctx, a)
-		if err != nil {
-			log.Warn().Err(err).Str("artist", a.Name).Msg("failed to build album details, returning artist without albums")
-		}
+	artists := make([]*pb.ArtistSummary, len(dbArtists))
+	var mu sync.Mutex
+	g, gCtx := errgroup.WithContext(ctx)
+	g.SetLimit(5)
 
-		artists = append(artists, &pb.ArtistSummary{
-			Id:           a.ID,
-			ExternalId:   a.ExternalID,
-			Name:         a.Name,
-			ArtistType:   a.ArtistType,
-			Country:      a.Country,
-			Genres:       a.Genres,
-			ImageUrl:     a.ImageURL,
-			MonitorState: toProtoMonitorState(a.MonitorState),
-			Albums:       albums,
+	for i, a := range dbArtists {
+		i, a := i, a
+		g.Go(func() error {
+			albums, err := service.buildAlbumsForArtist(gCtx, a)
+			if err != nil {
+				log.Warn().Err(err).Str("artist", a.Name).Msg("failed to build album details, returning artist without albums")
+			}
+
+			summary := &pb.ArtistSummary{
+				Id:           a.ID,
+				ExternalId:   a.ExternalID,
+				Name:         a.Name,
+				ArtistType:   a.ArtistType,
+				Country:      a.Country,
+				Genres:       a.Genres,
+				ImageUrl:     a.ImageURL,
+				MonitorState: toProtoMonitorState(a.MonitorState),
+				Albums:       albums,
+			}
+
+			mu.Lock()
+			artists[i] = summary
+			mu.Unlock()
+			return nil
 		})
 	}
 
+	_ = g.Wait()
+
 	return &pb.GetArtistsResponse{Artists: artists}, nil
 }
 
 func (service *MusicAgregatorService) buildAlbumsForArtist(ctx context.Context, artist *database.Artist) ([]*pb.AlbumDetail, error) {
-	metadataAlbums, err := service.metadata.GetArtistAlbums(ctx, artist.ExternalID)
+	var metadataAlbums []*metadataPb.Album
+	var err error
+	for attempt := 0; attempt < 3; attempt++ {
+		metadataAlbums, err = service.metadata.GetArtistAlbums(ctx, artist.ExternalID)
+		if err == nil {
+			break
+		}
+		log.Warn().Err(err).Int("attempt", attempt+1).Str("artist", artist.Name).Msg("metadata GetArtistAlbums failed, retrying")
+		time.Sleep(time.Duration(attempt+1) * 200 * time.Millisecond)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("fetching metadata albums: %w", err)
 	}
 
-	for _, ma := range metadataAlbums {
-		service.metadata.PersistAlbumForArtist(ctx, ma, artist.ID, database.Unmonitored)
-	}
+	service.metadata.PersistAlbumsForArtist(ctx, metadataAlbums, artist.ID, database.Unmonitored)
 
 	dbAlbums, err := service.metadata.GetAlbumsByArtistID(ctx, artist.ID)
 	if err != nil {
@@ -165,10 +188,14 @@ func (service *MusicAgregatorService) buildAlbumsForArtist(ctx context.Context,
 	}
 
 	dbAlbumsByExternalID := make(map[string]*database.Album, len(dbAlbums))
+	albumIDs := make([]string, 0, len(dbAlbums))
 	for _, a := range dbAlbums {
 		dbAlbumsByExternalID[a.ExternalID] = a
+		albumIDs = append(albumIDs, a.ID)
 	}
 
+	downloadsByAlbumID, _ := service.downloads.GetLatestByAlbumIDs(ctx, albumIDs)
+
 	albums := make([]*pb.AlbumDetail, 0, len(metadataAlbums))
 	for _, ma := range metadataAlbums {
 		detail := &pb.AlbumDetail{
@@ -192,14 +219,12 @@ func (service *MusicAgregatorService) buildAlbumsForArtist(ctx context.Context,
 			detail.Id = dbAlbum.ID
 			detail.MonitorState = toProtoMonitorState(dbAlbum.MonitorState)
 
-			downloads, err := service.downloads.GetByAlbumID(ctx, dbAlbum.ID)
-			if err == nil && len(downloads) > 0 {
-				best := downloads[0]
+			if d, ok := downloadsByAlbumID[dbAlbum.ID]; ok {
 				detail.Download = &pb.DownloadInfo{
-					State:    best.State,
-					Format:   best.Format,
-					Quality:  best.Quality,
-					SavePath: derefStr(best.SavePath),
+					State:    d.State,
+					Format:   d.Format,
+					Quality:  d.Quality,
+					SavePath: derefStr(d.SavePath),
 				}
 			}
 		} else {