feat: fuzzy search with parallel DB/MusicBrainz execution

- Add pg_trgm fuzzy search to PostgreSQL artist repository
- Add Lucene fuzzy query (~0.7) for MusicBrainz artist search
- Run DB and MusicBrainz searches in parallel goroutines
- Cache MusicBrainz results to DB for accumulation
- Deduplicate results by external ID
- Filter out special purpose artists (type=Other)
- Add SaveAll method for batch artist inserts
- Move database schema to containers repo
This commit is contained in:
Alexander
2026-05-10 21:13:17 +02:00
parent b4e763be44
commit 2756cc1043
9 changed files with 277 additions and 263 deletions
+4
View File
@@ -25,6 +25,10 @@ func (r *noopArtistRepo) Save(ctx context.Context, artist *domain.Artist) error
return nil
}
func (r *noopArtistRepo) SaveAll(ctx context.Context, artists []domain.Artist) error {
return nil
}
type noopAlbumRepo struct{}
func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {
-23
View File
@@ -1,23 +0,0 @@
services:
postgres:
image: postgres:16-alpine
container_name: metadata-postgres
environment:
POSTGRES_USER: metadata
POSTGRES_PASSWORD: metadata
POSTGRES_DB: metadata
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
- ./migrations:/docker-entrypoint-initdb.d:ro
- ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
command: postgres -c config_file=/etc/postgresql/postgresql.conf
healthcheck:
test: ["CMD-SHELL", "pg_isready -U metadata -d metadata"]
interval: 5s
timeout: 5s
retries: 5
volumes:
postgres_data:
-10
View File
@@ -1,10 +0,0 @@
listen_addresses = '*'
shared_preload_libraries = 'pg_prewarm'
pg_prewarm.autoprewarm = true
pg_prewarm.autoprewarm_interval = 300
shared_buffers = 256MB
effective_cache_size = 768MB
work_mem = 16MB
maintenance_work_mem = 128MB
-192
View File
@@ -1,192 +0,0 @@
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
CREATE TABLE artists (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
sort_name TEXT,
artist_type TEXT,
country TEXT,
formed_date DATE,
disbanded_date DATE,
description TEXT,
image_url TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE works (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
title TEXT NOT NULL,
work_type TEXT,
language TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE tracks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
work_id UUID REFERENCES works(id),
title TEXT NOT NULL,
duration_ms INT,
isrc TEXT,
explicit BOOLEAN DEFAULT false,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE labels (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
country TEXT,
founded_date DATE,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE albums (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
label_id UUID REFERENCES labels(id),
title TEXT NOT NULL,
album_type TEXT,
secondary_types TEXT[] DEFAULT '{}',
release_date DATE,
upc TEXT,
total_tracks INT,
total_discs INT DEFAULT 1,
cover_url TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE genres (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL UNIQUE,
parent_id UUID REFERENCES genres(id)
);
CREATE TABLE track_artists (
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
role TEXT DEFAULT 'primary',
position INT DEFAULT 0,
PRIMARY KEY (track_id, artist_id, role)
);
CREATE TABLE album_artists (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
role TEXT DEFAULT 'primary',
position INT DEFAULT 0,
PRIMARY KEY (album_id, artist_id, role)
);
CREATE TABLE album_tracks (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
disc_number INT DEFAULT 1,
track_number INT NOT NULL,
PRIMARY KEY (album_id, track_id)
);
CREATE TABLE work_artists (
work_id UUID REFERENCES works(id) ON DELETE CASCADE,
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
role TEXT DEFAULT 'writer',
PRIMARY KEY (work_id, artist_id, role)
);
CREATE TABLE artist_genres (
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
genre_id UUID REFERENCES genres(id) ON DELETE CASCADE,
PRIMARY KEY (artist_id, genre_id)
);
CREATE TABLE album_genres (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
genre_id UUID REFERENCES genres(id) ON DELETE CASCADE,
PRIMARY KEY (album_id, genre_id)
);
CREATE TABLE similar_artists (
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
similar_artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
score REAL DEFAULT 0.5,
PRIMARY KEY (artist_id, similar_artist_id)
);
CREATE TABLE lyrics (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
content TEXT,
synced_content JSONB,
language TEXT,
source TEXT NOT NULL,
source_id TEXT,
created_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE playlists (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name TEXT NOT NULL,
description TEXT,
is_public BOOLEAN DEFAULT true,
cover_url TEXT,
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
CREATE TABLE playlist_tracks (
playlist_id UUID REFERENCES playlists(id) ON DELETE CASCADE,
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
position INT NOT NULL,
added_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (playlist_id, track_id)
);
CREATE TABLE artist_external_ids (
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT,
fetched_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (artist_id, source, source_id)
);
CREATE TABLE album_external_ids (
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT,
fetched_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (album_id, source, source_id)
);
CREATE TABLE track_external_ids (
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
url TEXT,
fetched_at TIMESTAMPTZ DEFAULT now(),
PRIMARY KEY (track_id, source, source_id)
);
CREATE INDEX idx_artists_name ON artists(name);
CREATE INDEX idx_artists_source ON artists(source, source_id);
CREATE INDEX idx_tracks_isrc ON tracks(isrc) WHERE isrc IS NOT NULL;
CREATE INDEX idx_tracks_source ON tracks(source, source_id);
CREATE INDEX idx_albums_upc ON albums(upc) WHERE upc IS NOT NULL;
CREATE INDEX idx_albums_source ON albums(source, source_id);
CREATE INDEX idx_albums_release_date ON albums(release_date);
CREATE INDEX idx_genres_name ON genres(name);
CREATE INDEX idx_lyrics_track_id ON lyrics(track_id);
CREATE INDEX idx_playlist_tracks_position ON playlist_tracks(playlist_id, position);
+27 -2
View File
@@ -42,8 +42,8 @@ func (p *Provider) SearchArtists(ctx context.Context, query string, limit, offse
limit = 25
}
escapedQuery := escapeQuery(query)
data, err := p.client.search(ctx, "artist", fmt.Sprintf("artist:%s", escapedQuery), limit, offset)
luceneQuery := buildFuzzyArtistQuery(query, 0.7)
data, err := p.client.search(ctx, "artist", luceneQuery, limit, offset)
if err != nil {
return nil, fmt.Errorf("search artists: %w", err)
}
@@ -311,6 +311,31 @@ func selectCanonicalRelease(releases []*mbRelease) *mbRelease {
return best
}
func buildFuzzyArtistQuery(query string, similarity float64) string {
terms := strings.Fields(query)
if len(terms) == 0 {
return ""
}
var parts []string
for _, term := range terms {
escaped := escapeForFuzzy(term)
parts = append(parts, fmt.Sprintf("%s~%.1f", escaped, similarity))
}
termQuery := strings.Join(parts, " AND ")
return fmt.Sprintf("(artist:(%s) OR alias:(%s))", termQuery, termQuery)
}
func escapeForFuzzy(s string) string {
special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `:`, `/`, `\`}
result := s
for _, char := range special {
result = strings.ReplaceAll(result, char, `\`+char)
}
return result
}
func escapeQuery(s string) string {
special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `~`, `*`, `?`, `:`, `/`, `\`}
result := s
+129 -28
View File
@@ -60,23 +60,31 @@ func (r *ArtistRepository) GetByExternalID(ctx context.Context, source, sourceID
}
func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
countQuery := `SELECT COUNT(*) FROM artists WHERE name ILIKE $1`
// Use pg_trgm similarity for fuzzy matching with a threshold of 0.3
// Combined with ILIKE for substring matches
countQuery := `
SELECT COUNT(*) FROM artists
WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3`
searchQuery := `
SELECT id, name, sort_name, artist_type, country, formed_date, disbanded_date,
description, image_url, source, source_id
description, image_url, source, source_id,
GREATEST(
similarity(lower(name), lower($2)),
CASE WHEN name ILIKE $1 THEN 0.5 ELSE 0 END
) as score
FROM artists
WHERE name ILIKE $1
ORDER BY name
LIMIT $2 OFFSET $3`
WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3
ORDER BY score DESC, name
LIMIT $3 OFFSET $4`
pattern := "%" + query + "%"
var total int
if err := r.pool.QueryRow(ctx, countQuery, pattern).Scan(&total); err != nil {
if err := r.pool.QueryRow(ctx, countQuery, pattern, query).Scan(&total); err != nil {
return nil, err
}
rows, err := r.pool.Query(ctx, searchQuery, pattern, limit, offset)
rows, err := r.pool.Query(ctx, searchQuery, pattern, query, limit, offset)
if err != nil {
return nil, err
}
@@ -84,7 +92,7 @@ func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offs
var artists []domain.Artist
for rows.Next() {
artist, err := r.scanArtistFromRow(rows)
artist, err := r.scanArtistFromRowWithScore(rows)
if err != nil {
return nil, err
}
@@ -152,20 +160,79 @@ func (r *ArtistRepository) Save(ctx context.Context, artist *domain.Artist) erro
return tx.Commit(ctx)
}
func (r *ArtistRepository) SaveAll(ctx context.Context, artists []domain.Artist) error {
if len(artists) == 0 {
return nil
}
tx, err := r.pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx)
for _, artist := range artists {
var source, sourceID string
if len(artist.ExternalIDs) > 0 {
source = artist.ExternalIDs[0].Source
sourceID = artist.ExternalIDs[0].SourceID
}
query := `
INSERT INTO artists (id, name, sort_name, artist_type, country, formed_date,
disbanded_date, description, image_url, source, source_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
sort_name = EXCLUDED.sort_name,
artist_type = EXCLUDED.artist_type,
country = EXCLUDED.country,
formed_date = EXCLUDED.formed_date,
disbanded_date = EXCLUDED.disbanded_date,
description = EXCLUDED.description,
image_url = EXCLUDED.image_url,
updated_at = now()`
_, err = tx.Exec(ctx, query,
artist.ID, artist.Name, nullString(artist.SortName), nullString(artist.Type),
nullString(artist.Country), artist.FormedDate, artist.DisbandedDate,
nullString(artist.Description), nullString(artist.ImageURL), source, sourceID)
if err != nil {
return err
}
for _, ext := range artist.ExternalIDs {
extQuery := `
INSERT INTO artist_external_ids (artist_id, source, source_id, url)
VALUES ($1, $2, $3, $4)
ON CONFLICT (artist_id, source, source_id) DO UPDATE SET
url = EXCLUDED.url,
fetched_at = now()`
_, err = tx.Exec(ctx, extQuery, artist.ID, ext.Source, ext.SourceID, nullString(ext.URL))
if err != nil {
return err
}
}
}
return tx.Commit(ctx)
}
func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ...any) (*domain.Artist, error) {
row := r.pool.QueryRow(ctx, query, args...)
var (
artist domain.Artist
sortName *string
artistType *string
country *string
formedDate *time.Time
disbandDate *time.Time
description *string
imageURL *string
source string
sourceID *string
artist domain.Artist
sortName *string
artistType *string
country *string
formedDate *time.Time
disbandDate *time.Time
description *string
imageURL *string
source string
sourceID *string
)
err := row.Scan(
@@ -192,16 +259,16 @@ func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ..
func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error) {
var (
artist domain.Artist
sortName *string
artistType *string
country *string
formedDate *time.Time
disbandDate *time.Time
description *string
imageURL *string
source string
sourceID *string
artist domain.Artist
sortName *string
artistType *string
country *string
formedDate *time.Time
disbandDate *time.Time
description *string
imageURL *string
source string
sourceID *string
)
err := row.Scan(
@@ -223,6 +290,40 @@ func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error
return &artist, nil
}
func (r *ArtistRepository) scanArtistFromRowWithScore(row pgx.Row) (*domain.Artist, error) {
var (
artist domain.Artist
sortName *string
artistType *string
country *string
formedDate *time.Time
disbandDate *time.Time
description *string
imageURL *string
source string
sourceID *string
score float64
)
err := row.Scan(
&artist.ID, &artist.Name, &sortName, &artistType, &country,
&formedDate, &disbandDate, &description, &imageURL, &source, &sourceID, &score,
)
if err != nil {
return nil, err
}
artist.SortName = derefString(sortName)
artist.Type = derefString(artistType)
artist.Country = derefString(country)
artist.FormedDate = formedDate
artist.DisbandedDate = disbandDate
artist.Description = derefString(description)
artist.ImageURL = derefString(imageURL)
return &artist, nil
}
func (r *ArtistRepository) loadExternalIDs(ctx context.Context, artist *domain.Artist) error {
query := `SELECT source, source_id, url FROM artist_external_ids WHERE artist_id = $1`
+1
View File
@@ -11,6 +11,7 @@ type ArtistRepository interface {
GetByExternalID(ctx context.Context, source, sourceID string) (*domain.Artist, error)
Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error)
Save(ctx context.Context, artist *domain.Artist) error
SaveAll(ctx context.Context, artists []domain.Artist) error
}
type AlbumRepository interface {
+112 -8
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"errors"
"sync"
"github.com/rs/zerolog"
@@ -68,16 +69,119 @@ func (s *MetadataService) GetArtist(ctx context.Context, id string) (*domain.Art
func (s *MetadataService) SearchArtists(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
log := zerolog.Ctx(ctx)
result, err := s.artists.Search(ctx, query, limit, offset)
if err == nil && len(result.Items) > 0 {
log.Debug().Int("count", len(result.Items)).Msg("artist search served from cache")
metrics.CacheHits.WithLabelValues("artist_search").Inc()
return result, nil
var (
wg sync.WaitGroup
dbResult *domain.SearchResult[domain.Artist]
mbResult *domain.SearchResult[domain.Artist]
dbErr error
mbErr error
)
wg.Add(2)
go func() {
defer wg.Done()
dbResult, dbErr = s.artists.Search(ctx, query, limit, offset)
if dbErr == nil && len(dbResult.Items) > 0 {
metrics.CacheHits.WithLabelValues("artist_search").Inc()
}
}()
go func() {
defer wg.Done()
mbResult, mbErr = s.provider.SearchArtists(ctx, query, limit, offset)
if mbErr == nil && len(mbResult.Items) > 0 {
if saveErr := s.artists.SaveAll(ctx, mbResult.Items); saveErr != nil {
log.Warn().Err(saveErr).Msg("failed to cache provider search results")
} else {
log.Debug().Int("count", len(mbResult.Items)).Msg("cached provider search results")
}
}
}()
wg.Wait()
if dbErr != nil {
log.Warn().Err(dbErr).Msg("database search failed")
}
if mbErr != nil {
log.Warn().Err(mbErr).Msg("provider search failed")
}
metrics.CacheMisses.WithLabelValues("artist_search").Inc()
log.Debug().Str("query", query).Str("provider", s.provider.Name()).Msg("artist search cache miss, querying provider")
return s.provider.SearchArtists(ctx, query, limit, offset)
if dbErr != nil && mbErr != nil {
return nil, mbErr
}
combined := s.deduplicateArtists(dbResult, mbResult)
log.Debug().
Int("db_count", countItems(dbResult)).
Int("provider_count", countItems(mbResult)).
Int("combined_count", len(combined.Items)).
Msg("artist search completed")
return combined, nil
}
func (s *MetadataService) deduplicateArtists(dbResult, mbResult *domain.SearchResult[domain.Artist]) *domain.SearchResult[domain.Artist] {
seen := make(map[string]bool)
var items []domain.Artist
total := 0
addArtist := func(artist domain.Artist) {
if artist.Type == "Other" {
return
}
for _, ext := range artist.ExternalIDs {
key := ext.Source + ":" + ext.SourceID
if seen[key] {
return
}
}
for _, ext := range artist.ExternalIDs {
seen[ext.Source+":"+ext.SourceID] = true
}
items = append(items, artist)
}
if dbResult != nil {
for _, artist := range dbResult.Items {
addArtist(artist)
}
total = dbResult.Total
}
if mbResult != nil {
for _, artist := range mbResult.Items {
addArtist(artist)
}
if mbResult.Total > total {
total = mbResult.Total
}
}
limit := 25
offset := 0
if dbResult != nil {
limit = dbResult.Limit
offset = dbResult.Offset
} else if mbResult != nil {
limit = mbResult.Limit
offset = mbResult.Offset
}
return &domain.SearchResult[domain.Artist]{
Items: items,
Total: total,
Limit: limit,
Offset: offset,
}
}
func countItems[T any](result *domain.SearchResult[T]) int {
if result == nil {
return 0
}
return len(result.Items)
}
func (s *MetadataService) SearchAlbums(ctx context.Context, query string, artist string, limit, offset int) (*domain.SearchResult[domain.Album], error) {
+4
View File
@@ -25,6 +25,10 @@ func (r *noopArtistRepo) Save(ctx context.Context, artist *domain.Artist) error
return nil
}
func (r *noopArtistRepo) SaveAll(ctx context.Context, artists []domain.Artist) error {
return nil
}
type noopAlbumRepo struct{}
func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {