diff --git a/cmd/server/noop_repo.go b/cmd/server/noop_repo.go index da51cb8..068fe68 100644 --- a/cmd/server/noop_repo.go +++ b/cmd/server/noop_repo.go @@ -25,6 +25,10 @@ func (r *noopArtistRepo) Save(ctx context.Context, artist *domain.Artist) error return nil } +func (r *noopArtistRepo) SaveAll(ctx context.Context, artists []domain.Artist) error { + return nil +} + type noopAlbumRepo struct{} func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) { diff --git a/database/compose.yaml b/database/compose.yaml deleted file mode 100644 index defcacf..0000000 --- a/database/compose.yaml +++ /dev/null @@ -1,23 +0,0 @@ -services: - postgres: - image: postgres:16-alpine - container_name: metadata-postgres - environment: - POSTGRES_USER: metadata - POSTGRES_PASSWORD: metadata - POSTGRES_DB: metadata - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - ./migrations:/docker-entrypoint-initdb.d:ro - - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro - command: postgres -c config_file=/etc/postgresql/postgresql.conf - healthcheck: - test: ["CMD-SHELL", "pg_isready -U metadata -d metadata"] - interval: 5s - timeout: 5s - retries: 5 - -volumes: - postgres_data: diff --git a/database/postgresql.conf b/database/postgresql.conf deleted file mode 100644 index ebce120..0000000 --- a/database/postgresql.conf +++ /dev/null @@ -1,10 +0,0 @@ -listen_addresses = '*' -shared_preload_libraries = 'pg_prewarm' - -pg_prewarm.autoprewarm = true -pg_prewarm.autoprewarm_interval = 300 - -shared_buffers = 256MB -effective_cache_size = 768MB -work_mem = 16MB -maintenance_work_mem = 128MB diff --git a/database/schema.sql b/database/schema.sql deleted file mode 100644 index d88553b..0000000 --- a/database/schema.sql +++ /dev/null @@ -1,192 +0,0 @@ -CREATE EXTENSION IF NOT EXISTS pg_prewarm; - -CREATE TABLE artists ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name TEXT NOT NULL, - sort_name TEXT, - artist_type TEXT, - country TEXT, - formed_date DATE, - disbanded_date DATE, - description TEXT, - image_url TEXT, - source TEXT NOT NULL, - source_id TEXT, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE works ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - title TEXT NOT NULL, - work_type TEXT, - language TEXT, - source TEXT NOT NULL, - source_id TEXT, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE tracks ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - work_id UUID REFERENCES works(id), - title TEXT NOT NULL, - duration_ms INT, - isrc TEXT, - explicit BOOLEAN DEFAULT false, - source TEXT NOT NULL, - source_id TEXT, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE labels ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name TEXT NOT NULL, - country TEXT, - founded_date DATE, - source TEXT NOT NULL, - source_id TEXT, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE albums ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - label_id UUID REFERENCES labels(id), - title TEXT NOT NULL, - album_type TEXT, - secondary_types TEXT[] DEFAULT '{}', - release_date DATE, - upc TEXT, - total_tracks INT, - total_discs INT DEFAULT 1, - cover_url TEXT, - source TEXT NOT NULL, - source_id TEXT, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE genres ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name TEXT NOT NULL UNIQUE, - parent_id UUID REFERENCES genres(id) -); - -CREATE TABLE track_artists ( - track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, - artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - role TEXT DEFAULT 'primary', - position INT DEFAULT 0, - PRIMARY KEY (track_id, artist_id, role) -); - -CREATE TABLE album_artists ( - album_id UUID REFERENCES albums(id) ON DELETE CASCADE, - artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - role TEXT DEFAULT 'primary', - position INT DEFAULT 0, - PRIMARY KEY (album_id, artist_id, role) -); - -CREATE TABLE album_tracks ( - album_id UUID REFERENCES albums(id) ON DELETE CASCADE, - track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, - disc_number INT DEFAULT 1, - track_number INT NOT NULL, - PRIMARY KEY (album_id, track_id) -); - -CREATE TABLE work_artists ( - work_id UUID REFERENCES works(id) ON DELETE CASCADE, - artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - role TEXT DEFAULT 'writer', - PRIMARY KEY (work_id, artist_id, role) -); - -CREATE TABLE artist_genres ( - artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - genre_id UUID REFERENCES genres(id) ON DELETE CASCADE, - PRIMARY KEY (artist_id, genre_id) -); - -CREATE TABLE album_genres ( - album_id UUID REFERENCES albums(id) ON DELETE CASCADE, - genre_id UUID REFERENCES genres(id) ON DELETE CASCADE, - PRIMARY KEY (album_id, genre_id) -); - -CREATE TABLE similar_artists ( - artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - similar_artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - score REAL DEFAULT 0.5, - PRIMARY KEY (artist_id, similar_artist_id) -); - -CREATE TABLE lyrics ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, - content TEXT, - synced_content JSONB, - language TEXT, - source TEXT NOT NULL, - source_id TEXT, - created_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE playlists ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name TEXT NOT NULL, - description TEXT, - is_public BOOLEAN DEFAULT true, - cover_url TEXT, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE TABLE playlist_tracks ( - playlist_id UUID REFERENCES playlists(id) ON DELETE CASCADE, - track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, - position INT NOT NULL, - added_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (playlist_id, track_id) -); - -CREATE TABLE artist_external_ids ( - artist_id UUID REFERENCES artists(id) ON DELETE CASCADE, - source TEXT NOT NULL, - source_id TEXT NOT NULL, - url TEXT, - fetched_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (artist_id, source, source_id) -); - -CREATE TABLE album_external_ids ( - album_id UUID REFERENCES albums(id) ON DELETE CASCADE, - source TEXT NOT NULL, - source_id TEXT NOT NULL, - url TEXT, - fetched_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (album_id, source, source_id) -); - -CREATE TABLE track_external_ids ( - track_id UUID REFERENCES tracks(id) ON DELETE CASCADE, - source TEXT NOT NULL, - source_id TEXT NOT NULL, - url TEXT, - fetched_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (track_id, source, source_id) -); - -CREATE INDEX idx_artists_name ON artists(name); -CREATE INDEX idx_artists_source ON artists(source, source_id); -CREATE INDEX idx_tracks_isrc ON tracks(isrc) WHERE isrc IS NOT NULL; -CREATE INDEX idx_tracks_source ON tracks(source, source_id); -CREATE INDEX idx_albums_upc ON albums(upc) WHERE upc IS NOT NULL; -CREATE INDEX idx_albums_source ON albums(source, source_id); -CREATE INDEX idx_albums_release_date ON albums(release_date); -CREATE INDEX idx_genres_name ON genres(name); -CREATE INDEX idx_lyrics_track_id ON lyrics(track_id); -CREATE INDEX idx_playlist_tracks_position ON playlist_tracks(playlist_id, position); diff --git a/internal/provider/musicbrainz/provider.go b/internal/provider/musicbrainz/provider.go index 490a987..bda9431 100644 --- a/internal/provider/musicbrainz/provider.go +++ b/internal/provider/musicbrainz/provider.go @@ -42,8 +42,8 @@ func (p *Provider) SearchArtists(ctx context.Context, query string, limit, offse limit = 25 } - escapedQuery := escapeQuery(query) - data, err := p.client.search(ctx, "artist", fmt.Sprintf("artist:%s", escapedQuery), limit, offset) + luceneQuery := buildFuzzyArtistQuery(query, 0.7) + data, err := p.client.search(ctx, "artist", luceneQuery, limit, offset) if err != nil { return nil, fmt.Errorf("search artists: %w", err) } @@ -311,6 +311,31 @@ func selectCanonicalRelease(releases []*mbRelease) *mbRelease { return best } +func buildFuzzyArtistQuery(query string, similarity float64) string { + terms := strings.Fields(query) + if len(terms) == 0 { + return "" + } + + var parts []string + for _, term := range terms { + escaped := escapeForFuzzy(term) + parts = append(parts, fmt.Sprintf("%s~%.1f", escaped, similarity)) + } + + termQuery := strings.Join(parts, " AND ") + return fmt.Sprintf("(artist:(%s) OR alias:(%s))", termQuery, termQuery) +} + +func escapeForFuzzy(s string) string { + special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `:`, `/`, `\`} + result := s + for _, char := range special { + result = strings.ReplaceAll(result, char, `\`+char) + } + return result +} + func escapeQuery(s string) string { special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `~`, `*`, `?`, `:`, `/`, `\`} result := s diff --git a/internal/repository/postgres/artist.go b/internal/repository/postgres/artist.go index 0c0524d..3e13e86 100644 --- a/internal/repository/postgres/artist.go +++ b/internal/repository/postgres/artist.go @@ -60,23 +60,31 @@ func (r *ArtistRepository) GetByExternalID(ctx context.Context, source, sourceID } func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) { - countQuery := `SELECT COUNT(*) FROM artists WHERE name ILIKE $1` + // Use pg_trgm similarity for fuzzy matching with a threshold of 0.3 + // Combined with ILIKE for substring matches + countQuery := ` + SELECT COUNT(*) FROM artists + WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3` searchQuery := ` SELECT id, name, sort_name, artist_type, country, formed_date, disbanded_date, - description, image_url, source, source_id + description, image_url, source, source_id, + GREATEST( + similarity(lower(name), lower($2)), + CASE WHEN name ILIKE $1 THEN 0.5 ELSE 0 END + ) as score FROM artists - WHERE name ILIKE $1 - ORDER BY name - LIMIT $2 OFFSET $3` + WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3 + ORDER BY score DESC, name + LIMIT $3 OFFSET $4` pattern := "%" + query + "%" var total int - if err := r.pool.QueryRow(ctx, countQuery, pattern).Scan(&total); err != nil { + if err := r.pool.QueryRow(ctx, countQuery, pattern, query).Scan(&total); err != nil { return nil, err } - rows, err := r.pool.Query(ctx, searchQuery, pattern, limit, offset) + rows, err := r.pool.Query(ctx, searchQuery, pattern, query, limit, offset) if err != nil { return nil, err } @@ -84,7 +92,7 @@ func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offs var artists []domain.Artist for rows.Next() { - artist, err := r.scanArtistFromRow(rows) + artist, err := r.scanArtistFromRowWithScore(rows) if err != nil { return nil, err } @@ -152,20 +160,79 @@ func (r *ArtistRepository) Save(ctx context.Context, artist *domain.Artist) erro return tx.Commit(ctx) } +func (r *ArtistRepository) SaveAll(ctx context.Context, artists []domain.Artist) error { + if len(artists) == 0 { + return nil + } + + tx, err := r.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + for _, artist := range artists { + var source, sourceID string + if len(artist.ExternalIDs) > 0 { + source = artist.ExternalIDs[0].Source + sourceID = artist.ExternalIDs[0].SourceID + } + + query := ` + INSERT INTO artists (id, name, sort_name, artist_type, country, formed_date, + disbanded_date, description, image_url, source, source_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (id) DO UPDATE SET + name = EXCLUDED.name, + sort_name = EXCLUDED.sort_name, + artist_type = EXCLUDED.artist_type, + country = EXCLUDED.country, + formed_date = EXCLUDED.formed_date, + disbanded_date = EXCLUDED.disbanded_date, + description = EXCLUDED.description, + image_url = EXCLUDED.image_url, + updated_at = now()` + + _, err = tx.Exec(ctx, query, + artist.ID, artist.Name, nullString(artist.SortName), nullString(artist.Type), + nullString(artist.Country), artist.FormedDate, artist.DisbandedDate, + nullString(artist.Description), nullString(artist.ImageURL), source, sourceID) + if err != nil { + return err + } + + for _, ext := range artist.ExternalIDs { + extQuery := ` + INSERT INTO artist_external_ids (artist_id, source, source_id, url) + VALUES ($1, $2, $3, $4) + ON CONFLICT (artist_id, source, source_id) DO UPDATE SET + url = EXCLUDED.url, + fetched_at = now()` + + _, err = tx.Exec(ctx, extQuery, artist.ID, ext.Source, ext.SourceID, nullString(ext.URL)) + if err != nil { + return err + } + } + } + + return tx.Commit(ctx) +} + func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ...any) (*domain.Artist, error) { row := r.pool.QueryRow(ctx, query, args...) var ( - artist domain.Artist - sortName *string - artistType *string - country *string - formedDate *time.Time - disbandDate *time.Time - description *string - imageURL *string - source string - sourceID *string + artist domain.Artist + sortName *string + artistType *string + country *string + formedDate *time.Time + disbandDate *time.Time + description *string + imageURL *string + source string + sourceID *string ) err := row.Scan( @@ -192,16 +259,16 @@ func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args .. func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error) { var ( - artist domain.Artist - sortName *string - artistType *string - country *string - formedDate *time.Time - disbandDate *time.Time - description *string - imageURL *string - source string - sourceID *string + artist domain.Artist + sortName *string + artistType *string + country *string + formedDate *time.Time + disbandDate *time.Time + description *string + imageURL *string + source string + sourceID *string ) err := row.Scan( @@ -223,6 +290,40 @@ func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error return &artist, nil } +func (r *ArtistRepository) scanArtistFromRowWithScore(row pgx.Row) (*domain.Artist, error) { + var ( + artist domain.Artist + sortName *string + artistType *string + country *string + formedDate *time.Time + disbandDate *time.Time + description *string + imageURL *string + source string + sourceID *string + score float64 + ) + + err := row.Scan( + &artist.ID, &artist.Name, &sortName, &artistType, &country, + &formedDate, &disbandDate, &description, &imageURL, &source, &sourceID, &score, + ) + if err != nil { + return nil, err + } + + artist.SortName = derefString(sortName) + artist.Type = derefString(artistType) + artist.Country = derefString(country) + artist.FormedDate = formedDate + artist.DisbandedDate = disbandDate + artist.Description = derefString(description) + artist.ImageURL = derefString(imageURL) + + return &artist, nil +} + func (r *ArtistRepository) loadExternalIDs(ctx context.Context, artist *domain.Artist) error { query := `SELECT source, source_id, url FROM artist_external_ids WHERE artist_id = $1` diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 2c983a6..c91df31 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -11,6 +11,7 @@ type ArtistRepository interface { GetByExternalID(ctx context.Context, source, sourceID string) (*domain.Artist, error) Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) Save(ctx context.Context, artist *domain.Artist) error + SaveAll(ctx context.Context, artists []domain.Artist) error } type AlbumRepository interface { diff --git a/internal/service/metadata.go b/internal/service/metadata.go index c2e32c4..a0efb3b 100644 --- a/internal/service/metadata.go +++ b/internal/service/metadata.go @@ -3,6 +3,7 @@ package service import ( "context" "errors" + "sync" "github.com/rs/zerolog" @@ -68,16 +69,119 @@ func (s *MetadataService) GetArtist(ctx context.Context, id string) (*domain.Art func (s *MetadataService) SearchArtists(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) { log := zerolog.Ctx(ctx) - result, err := s.artists.Search(ctx, query, limit, offset) - if err == nil && len(result.Items) > 0 { - log.Debug().Int("count", len(result.Items)).Msg("artist search served from cache") - metrics.CacheHits.WithLabelValues("artist_search").Inc() - return result, nil + var ( + wg sync.WaitGroup + dbResult *domain.SearchResult[domain.Artist] + mbResult *domain.SearchResult[domain.Artist] + dbErr error + mbErr error + ) + + wg.Add(2) + + go func() { + defer wg.Done() + dbResult, dbErr = s.artists.Search(ctx, query, limit, offset) + if dbErr == nil && len(dbResult.Items) > 0 { + metrics.CacheHits.WithLabelValues("artist_search").Inc() + } + }() + + go func() { + defer wg.Done() + mbResult, mbErr = s.provider.SearchArtists(ctx, query, limit, offset) + if mbErr == nil && len(mbResult.Items) > 0 { + if saveErr := s.artists.SaveAll(ctx, mbResult.Items); saveErr != nil { + log.Warn().Err(saveErr).Msg("failed to cache provider search results") + } else { + log.Debug().Int("count", len(mbResult.Items)).Msg("cached provider search results") + } + } + }() + + wg.Wait() + + if dbErr != nil { + log.Warn().Err(dbErr).Msg("database search failed") + } + if mbErr != nil { + log.Warn().Err(mbErr).Msg("provider search failed") } - metrics.CacheMisses.WithLabelValues("artist_search").Inc() - log.Debug().Str("query", query).Str("provider", s.provider.Name()).Msg("artist search cache miss, querying provider") - return s.provider.SearchArtists(ctx, query, limit, offset) + if dbErr != nil && mbErr != nil { + return nil, mbErr + } + + combined := s.deduplicateArtists(dbResult, mbResult) + log.Debug(). + Int("db_count", countItems(dbResult)). + Int("provider_count", countItems(mbResult)). + Int("combined_count", len(combined.Items)). + Msg("artist search completed") + + return combined, nil +} + +func (s *MetadataService) deduplicateArtists(dbResult, mbResult *domain.SearchResult[domain.Artist]) *domain.SearchResult[domain.Artist] { + seen := make(map[string]bool) + var items []domain.Artist + total := 0 + + addArtist := func(artist domain.Artist) { + if artist.Type == "Other" { + return + } + for _, ext := range artist.ExternalIDs { + key := ext.Source + ":" + ext.SourceID + if seen[key] { + return + } + } + for _, ext := range artist.ExternalIDs { + seen[ext.Source+":"+ext.SourceID] = true + } + items = append(items, artist) + } + + if dbResult != nil { + for _, artist := range dbResult.Items { + addArtist(artist) + } + total = dbResult.Total + } + + if mbResult != nil { + for _, artist := range mbResult.Items { + addArtist(artist) + } + if mbResult.Total > total { + total = mbResult.Total + } + } + + limit := 25 + offset := 0 + if dbResult != nil { + limit = dbResult.Limit + offset = dbResult.Offset + } else if mbResult != nil { + limit = mbResult.Limit + offset = mbResult.Offset + } + + return &domain.SearchResult[domain.Artist]{ + Items: items, + Total: total, + Limit: limit, + Offset: offset, + } +} + +func countItems[T any](result *domain.SearchResult[T]) int { + if result == nil { + return 0 + } + return len(result.Items) } func (s *MetadataService) SearchAlbums(ctx context.Context, query string, artist string, limit, offset int) (*domain.SearchResult[domain.Album], error) { diff --git a/tests/e2e/noop_repo_test.go b/tests/e2e/noop_repo_test.go index 37eb4e7..a8b8e6d 100644 --- a/tests/e2e/noop_repo_test.go +++ b/tests/e2e/noop_repo_test.go @@ -25,6 +25,10 @@ func (r *noopArtistRepo) Save(ctx context.Context, artist *domain.Artist) error return nil } +func (r *noopArtistRepo) SaveAll(ctx context.Context, artists []domain.Artist) error { + return nil +} + type noopAlbumRepo struct{} func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {