feat: fuzzy search with parallel DB/MusicBrainz execution
- Add pg_trgm fuzzy search to PostgreSQL artist repository - Add Lucene fuzzy query (~0.7) for MusicBrainz artist search - Run DB and MusicBrainz searches in parallel goroutines - Cache MusicBrainz results to DB for accumulation - Deduplicate results by external ID - Filter out special purpose artists (type=Other) - Add SaveAll method for batch artist inserts - Move database schema to containers repo
This commit is contained in:
@@ -25,6 +25,10 @@ func (r *noopArtistRepo) Save(ctx context.Context, artist *domain.Artist) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *noopArtistRepo) SaveAll(ctx context.Context, artists []domain.Artist) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type noopAlbumRepo struct{}
|
type noopAlbumRepo struct{}
|
||||||
|
|
||||||
func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {
|
func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16-alpine
|
|
||||||
container_name: metadata-postgres
|
|
||||||
environment:
|
|
||||||
POSTGRES_USER: metadata
|
|
||||||
POSTGRES_PASSWORD: metadata
|
|
||||||
POSTGRES_DB: metadata
|
|
||||||
ports:
|
|
||||||
- "5432:5432"
|
|
||||||
volumes:
|
|
||||||
- postgres_data:/var/lib/postgresql/data
|
|
||||||
- ./migrations:/docker-entrypoint-initdb.d:ro
|
|
||||||
- ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
|
||||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "pg_isready -U metadata -d metadata"]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 5
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
postgres_data:
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
listen_addresses = '*'
|
|
||||||
shared_preload_libraries = 'pg_prewarm'
|
|
||||||
|
|
||||||
pg_prewarm.autoprewarm = true
|
|
||||||
pg_prewarm.autoprewarm_interval = 300
|
|
||||||
|
|
||||||
shared_buffers = 256MB
|
|
||||||
effective_cache_size = 768MB
|
|
||||||
work_mem = 16MB
|
|
||||||
maintenance_work_mem = 128MB
|
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
CREATE EXTENSION IF NOT EXISTS pg_prewarm;
|
|
||||||
|
|
||||||
CREATE TABLE artists (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
sort_name TEXT,
|
|
||||||
artist_type TEXT,
|
|
||||||
country TEXT,
|
|
||||||
formed_date DATE,
|
|
||||||
disbanded_date DATE,
|
|
||||||
description TEXT,
|
|
||||||
image_url TEXT,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE works (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
work_type TEXT,
|
|
||||||
language TEXT,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE tracks (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
work_id UUID REFERENCES works(id),
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
duration_ms INT,
|
|
||||||
isrc TEXT,
|
|
||||||
explicit BOOLEAN DEFAULT false,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE labels (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
country TEXT,
|
|
||||||
founded_date DATE,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE albums (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
label_id UUID REFERENCES labels(id),
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
album_type TEXT,
|
|
||||||
secondary_types TEXT[] DEFAULT '{}',
|
|
||||||
release_date DATE,
|
|
||||||
upc TEXT,
|
|
||||||
total_tracks INT,
|
|
||||||
total_discs INT DEFAULT 1,
|
|
||||||
cover_url TEXT,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE genres (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
name TEXT NOT NULL UNIQUE,
|
|
||||||
parent_id UUID REFERENCES genres(id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE track_artists (
|
|
||||||
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
|
|
||||||
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
role TEXT DEFAULT 'primary',
|
|
||||||
position INT DEFAULT 0,
|
|
||||||
PRIMARY KEY (track_id, artist_id, role)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE album_artists (
|
|
||||||
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
|
|
||||||
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
role TEXT DEFAULT 'primary',
|
|
||||||
position INT DEFAULT 0,
|
|
||||||
PRIMARY KEY (album_id, artist_id, role)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE album_tracks (
|
|
||||||
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
|
|
||||||
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
|
|
||||||
disc_number INT DEFAULT 1,
|
|
||||||
track_number INT NOT NULL,
|
|
||||||
PRIMARY KEY (album_id, track_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE work_artists (
|
|
||||||
work_id UUID REFERENCES works(id) ON DELETE CASCADE,
|
|
||||||
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
role TEXT DEFAULT 'writer',
|
|
||||||
PRIMARY KEY (work_id, artist_id, role)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE artist_genres (
|
|
||||||
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
genre_id UUID REFERENCES genres(id) ON DELETE CASCADE,
|
|
||||||
PRIMARY KEY (artist_id, genre_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE album_genres (
|
|
||||||
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
|
|
||||||
genre_id UUID REFERENCES genres(id) ON DELETE CASCADE,
|
|
||||||
PRIMARY KEY (album_id, genre_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE similar_artists (
|
|
||||||
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
similar_artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
score REAL DEFAULT 0.5,
|
|
||||||
PRIMARY KEY (artist_id, similar_artist_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE lyrics (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
|
|
||||||
content TEXT,
|
|
||||||
synced_content JSONB,
|
|
||||||
language TEXT,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE playlists (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
description TEXT,
|
|
||||||
is_public BOOLEAN DEFAULT true,
|
|
||||||
cover_url TEXT,
|
|
||||||
created_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
updated_at TIMESTAMPTZ DEFAULT now()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE playlist_tracks (
|
|
||||||
playlist_id UUID REFERENCES playlists(id) ON DELETE CASCADE,
|
|
||||||
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
|
|
||||||
position INT NOT NULL,
|
|
||||||
added_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
PRIMARY KEY (playlist_id, track_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE artist_external_ids (
|
|
||||||
artist_id UUID REFERENCES artists(id) ON DELETE CASCADE,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT NOT NULL,
|
|
||||||
url TEXT,
|
|
||||||
fetched_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
PRIMARY KEY (artist_id, source, source_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE album_external_ids (
|
|
||||||
album_id UUID REFERENCES albums(id) ON DELETE CASCADE,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT NOT NULL,
|
|
||||||
url TEXT,
|
|
||||||
fetched_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
PRIMARY KEY (album_id, source, source_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE track_external_ids (
|
|
||||||
track_id UUID REFERENCES tracks(id) ON DELETE CASCADE,
|
|
||||||
source TEXT NOT NULL,
|
|
||||||
source_id TEXT NOT NULL,
|
|
||||||
url TEXT,
|
|
||||||
fetched_at TIMESTAMPTZ DEFAULT now(),
|
|
||||||
PRIMARY KEY (track_id, source, source_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX idx_artists_name ON artists(name);
|
|
||||||
CREATE INDEX idx_artists_source ON artists(source, source_id);
|
|
||||||
CREATE INDEX idx_tracks_isrc ON tracks(isrc) WHERE isrc IS NOT NULL;
|
|
||||||
CREATE INDEX idx_tracks_source ON tracks(source, source_id);
|
|
||||||
CREATE INDEX idx_albums_upc ON albums(upc) WHERE upc IS NOT NULL;
|
|
||||||
CREATE INDEX idx_albums_source ON albums(source, source_id);
|
|
||||||
CREATE INDEX idx_albums_release_date ON albums(release_date);
|
|
||||||
CREATE INDEX idx_genres_name ON genres(name);
|
|
||||||
CREATE INDEX idx_lyrics_track_id ON lyrics(track_id);
|
|
||||||
CREATE INDEX idx_playlist_tracks_position ON playlist_tracks(playlist_id, position);
|
|
||||||
@@ -42,8 +42,8 @@ func (p *Provider) SearchArtists(ctx context.Context, query string, limit, offse
|
|||||||
limit = 25
|
limit = 25
|
||||||
}
|
}
|
||||||
|
|
||||||
escapedQuery := escapeQuery(query)
|
luceneQuery := buildFuzzyArtistQuery(query, 0.7)
|
||||||
data, err := p.client.search(ctx, "artist", fmt.Sprintf("artist:%s", escapedQuery), limit, offset)
|
data, err := p.client.search(ctx, "artist", luceneQuery, limit, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("search artists: %w", err)
|
return nil, fmt.Errorf("search artists: %w", err)
|
||||||
}
|
}
|
||||||
@@ -311,6 +311,31 @@ func selectCanonicalRelease(releases []*mbRelease) *mbRelease {
|
|||||||
return best
|
return best
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func buildFuzzyArtistQuery(query string, similarity float64) string {
|
||||||
|
terms := strings.Fields(query)
|
||||||
|
if len(terms) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var parts []string
|
||||||
|
for _, term := range terms {
|
||||||
|
escaped := escapeForFuzzy(term)
|
||||||
|
parts = append(parts, fmt.Sprintf("%s~%.1f", escaped, similarity))
|
||||||
|
}
|
||||||
|
|
||||||
|
termQuery := strings.Join(parts, " AND ")
|
||||||
|
return fmt.Sprintf("(artist:(%s) OR alias:(%s))", termQuery, termQuery)
|
||||||
|
}
|
||||||
|
|
||||||
|
func escapeForFuzzy(s string) string {
|
||||||
|
special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `:`, `/`, `\`}
|
||||||
|
result := s
|
||||||
|
for _, char := range special {
|
||||||
|
result = strings.ReplaceAll(result, char, `\`+char)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func escapeQuery(s string) string {
|
func escapeQuery(s string) string {
|
||||||
special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `~`, `*`, `?`, `:`, `/`, `\`}
|
special := []string{`+`, `-`, `&`, `|`, `!`, `(`, `)`, `{`, `}`, `[`, `]`, `^`, `"`, `~`, `*`, `?`, `:`, `/`, `\`}
|
||||||
result := s
|
result := s
|
||||||
|
|||||||
@@ -60,23 +60,31 @@ func (r *ArtistRepository) GetByExternalID(ctx context.Context, source, sourceID
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
|
func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
|
||||||
countQuery := `SELECT COUNT(*) FROM artists WHERE name ILIKE $1`
|
// Use pg_trgm similarity for fuzzy matching with a threshold of 0.3
|
||||||
|
// Combined with ILIKE for substring matches
|
||||||
|
countQuery := `
|
||||||
|
SELECT COUNT(*) FROM artists
|
||||||
|
WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3`
|
||||||
searchQuery := `
|
searchQuery := `
|
||||||
SELECT id, name, sort_name, artist_type, country, formed_date, disbanded_date,
|
SELECT id, name, sort_name, artist_type, country, formed_date, disbanded_date,
|
||||||
description, image_url, source, source_id
|
description, image_url, source, source_id,
|
||||||
|
GREATEST(
|
||||||
|
similarity(lower(name), lower($2)),
|
||||||
|
CASE WHEN name ILIKE $1 THEN 0.5 ELSE 0 END
|
||||||
|
) as score
|
||||||
FROM artists
|
FROM artists
|
||||||
WHERE name ILIKE $1
|
WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3
|
||||||
ORDER BY name
|
ORDER BY score DESC, name
|
||||||
LIMIT $2 OFFSET $3`
|
LIMIT $3 OFFSET $4`
|
||||||
|
|
||||||
pattern := "%" + query + "%"
|
pattern := "%" + query + "%"
|
||||||
|
|
||||||
var total int
|
var total int
|
||||||
if err := r.pool.QueryRow(ctx, countQuery, pattern).Scan(&total); err != nil {
|
if err := r.pool.QueryRow(ctx, countQuery, pattern, query).Scan(&total); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
rows, err := r.pool.Query(ctx, searchQuery, pattern, limit, offset)
|
rows, err := r.pool.Query(ctx, searchQuery, pattern, query, limit, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -84,7 +92,7 @@ func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offs
|
|||||||
|
|
||||||
var artists []domain.Artist
|
var artists []domain.Artist
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
artist, err := r.scanArtistFromRow(rows)
|
artist, err := r.scanArtistFromRowWithScore(rows)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -152,20 +160,79 @@ func (r *ArtistRepository) Save(ctx context.Context, artist *domain.Artist) erro
|
|||||||
return tx.Commit(ctx)
|
return tx.Commit(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *ArtistRepository) SaveAll(ctx context.Context, artists []domain.Artist) error {
|
||||||
|
if len(artists) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := r.pool.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback(ctx)
|
||||||
|
|
||||||
|
for _, artist := range artists {
|
||||||
|
var source, sourceID string
|
||||||
|
if len(artist.ExternalIDs) > 0 {
|
||||||
|
source = artist.ExternalIDs[0].Source
|
||||||
|
sourceID = artist.ExternalIDs[0].SourceID
|
||||||
|
}
|
||||||
|
|
||||||
|
query := `
|
||||||
|
INSERT INTO artists (id, name, sort_name, artist_type, country, formed_date,
|
||||||
|
disbanded_date, description, image_url, source, source_id)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||||
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
|
name = EXCLUDED.name,
|
||||||
|
sort_name = EXCLUDED.sort_name,
|
||||||
|
artist_type = EXCLUDED.artist_type,
|
||||||
|
country = EXCLUDED.country,
|
||||||
|
formed_date = EXCLUDED.formed_date,
|
||||||
|
disbanded_date = EXCLUDED.disbanded_date,
|
||||||
|
description = EXCLUDED.description,
|
||||||
|
image_url = EXCLUDED.image_url,
|
||||||
|
updated_at = now()`
|
||||||
|
|
||||||
|
_, err = tx.Exec(ctx, query,
|
||||||
|
artist.ID, artist.Name, nullString(artist.SortName), nullString(artist.Type),
|
||||||
|
nullString(artist.Country), artist.FormedDate, artist.DisbandedDate,
|
||||||
|
nullString(artist.Description), nullString(artist.ImageURL), source, sourceID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ext := range artist.ExternalIDs {
|
||||||
|
extQuery := `
|
||||||
|
INSERT INTO artist_external_ids (artist_id, source, source_id, url)
|
||||||
|
VALUES ($1, $2, $3, $4)
|
||||||
|
ON CONFLICT (artist_id, source, source_id) DO UPDATE SET
|
||||||
|
url = EXCLUDED.url,
|
||||||
|
fetched_at = now()`
|
||||||
|
|
||||||
|
_, err = tx.Exec(ctx, extQuery, artist.ID, ext.Source, ext.SourceID, nullString(ext.URL))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return tx.Commit(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ...any) (*domain.Artist, error) {
|
func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ...any) (*domain.Artist, error) {
|
||||||
row := r.pool.QueryRow(ctx, query, args...)
|
row := r.pool.QueryRow(ctx, query, args...)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
artist domain.Artist
|
artist domain.Artist
|
||||||
sortName *string
|
sortName *string
|
||||||
artistType *string
|
artistType *string
|
||||||
country *string
|
country *string
|
||||||
formedDate *time.Time
|
formedDate *time.Time
|
||||||
disbandDate *time.Time
|
disbandDate *time.Time
|
||||||
description *string
|
description *string
|
||||||
imageURL *string
|
imageURL *string
|
||||||
source string
|
source string
|
||||||
sourceID *string
|
sourceID *string
|
||||||
)
|
)
|
||||||
|
|
||||||
err := row.Scan(
|
err := row.Scan(
|
||||||
@@ -192,16 +259,16 @@ func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ..
|
|||||||
|
|
||||||
func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error) {
|
func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error) {
|
||||||
var (
|
var (
|
||||||
artist domain.Artist
|
artist domain.Artist
|
||||||
sortName *string
|
sortName *string
|
||||||
artistType *string
|
artistType *string
|
||||||
country *string
|
country *string
|
||||||
formedDate *time.Time
|
formedDate *time.Time
|
||||||
disbandDate *time.Time
|
disbandDate *time.Time
|
||||||
description *string
|
description *string
|
||||||
imageURL *string
|
imageURL *string
|
||||||
source string
|
source string
|
||||||
sourceID *string
|
sourceID *string
|
||||||
)
|
)
|
||||||
|
|
||||||
err := row.Scan(
|
err := row.Scan(
|
||||||
@@ -223,6 +290,40 @@ func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error
|
|||||||
return &artist, nil
|
return &artist, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *ArtistRepository) scanArtistFromRowWithScore(row pgx.Row) (*domain.Artist, error) {
|
||||||
|
var (
|
||||||
|
artist domain.Artist
|
||||||
|
sortName *string
|
||||||
|
artistType *string
|
||||||
|
country *string
|
||||||
|
formedDate *time.Time
|
||||||
|
disbandDate *time.Time
|
||||||
|
description *string
|
||||||
|
imageURL *string
|
||||||
|
source string
|
||||||
|
sourceID *string
|
||||||
|
score float64
|
||||||
|
)
|
||||||
|
|
||||||
|
err := row.Scan(
|
||||||
|
&artist.ID, &artist.Name, &sortName, &artistType, &country,
|
||||||
|
&formedDate, &disbandDate, &description, &imageURL, &source, &sourceID, &score,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
artist.SortName = derefString(sortName)
|
||||||
|
artist.Type = derefString(artistType)
|
||||||
|
artist.Country = derefString(country)
|
||||||
|
artist.FormedDate = formedDate
|
||||||
|
artist.DisbandedDate = disbandDate
|
||||||
|
artist.Description = derefString(description)
|
||||||
|
artist.ImageURL = derefString(imageURL)
|
||||||
|
|
||||||
|
return &artist, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *ArtistRepository) loadExternalIDs(ctx context.Context, artist *domain.Artist) error {
|
func (r *ArtistRepository) loadExternalIDs(ctx context.Context, artist *domain.Artist) error {
|
||||||
query := `SELECT source, source_id, url FROM artist_external_ids WHERE artist_id = $1`
|
query := `SELECT source, source_id, url FROM artist_external_ids WHERE artist_id = $1`
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ type ArtistRepository interface {
|
|||||||
GetByExternalID(ctx context.Context, source, sourceID string) (*domain.Artist, error)
|
GetByExternalID(ctx context.Context, source, sourceID string) (*domain.Artist, error)
|
||||||
Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error)
|
Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error)
|
||||||
Save(ctx context.Context, artist *domain.Artist) error
|
Save(ctx context.Context, artist *domain.Artist) error
|
||||||
|
SaveAll(ctx context.Context, artists []domain.Artist) error
|
||||||
}
|
}
|
||||||
|
|
||||||
type AlbumRepository interface {
|
type AlbumRepository interface {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package service
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
|
|
||||||
"github.com/rs/zerolog"
|
"github.com/rs/zerolog"
|
||||||
|
|
||||||
@@ -68,16 +69,119 @@ func (s *MetadataService) GetArtist(ctx context.Context, id string) (*domain.Art
|
|||||||
func (s *MetadataService) SearchArtists(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
|
func (s *MetadataService) SearchArtists(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
|
||||||
log := zerolog.Ctx(ctx)
|
log := zerolog.Ctx(ctx)
|
||||||
|
|
||||||
result, err := s.artists.Search(ctx, query, limit, offset)
|
var (
|
||||||
if err == nil && len(result.Items) > 0 {
|
wg sync.WaitGroup
|
||||||
log.Debug().Int("count", len(result.Items)).Msg("artist search served from cache")
|
dbResult *domain.SearchResult[domain.Artist]
|
||||||
metrics.CacheHits.WithLabelValues("artist_search").Inc()
|
mbResult *domain.SearchResult[domain.Artist]
|
||||||
return result, nil
|
dbErr error
|
||||||
|
mbErr error
|
||||||
|
)
|
||||||
|
|
||||||
|
wg.Add(2)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
dbResult, dbErr = s.artists.Search(ctx, query, limit, offset)
|
||||||
|
if dbErr == nil && len(dbResult.Items) > 0 {
|
||||||
|
metrics.CacheHits.WithLabelValues("artist_search").Inc()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
mbResult, mbErr = s.provider.SearchArtists(ctx, query, limit, offset)
|
||||||
|
if mbErr == nil && len(mbResult.Items) > 0 {
|
||||||
|
if saveErr := s.artists.SaveAll(ctx, mbResult.Items); saveErr != nil {
|
||||||
|
log.Warn().Err(saveErr).Msg("failed to cache provider search results")
|
||||||
|
} else {
|
||||||
|
log.Debug().Int("count", len(mbResult.Items)).Msg("cached provider search results")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if dbErr != nil {
|
||||||
|
log.Warn().Err(dbErr).Msg("database search failed")
|
||||||
|
}
|
||||||
|
if mbErr != nil {
|
||||||
|
log.Warn().Err(mbErr).Msg("provider search failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics.CacheMisses.WithLabelValues("artist_search").Inc()
|
if dbErr != nil && mbErr != nil {
|
||||||
log.Debug().Str("query", query).Str("provider", s.provider.Name()).Msg("artist search cache miss, querying provider")
|
return nil, mbErr
|
||||||
return s.provider.SearchArtists(ctx, query, limit, offset)
|
}
|
||||||
|
|
||||||
|
combined := s.deduplicateArtists(dbResult, mbResult)
|
||||||
|
log.Debug().
|
||||||
|
Int("db_count", countItems(dbResult)).
|
||||||
|
Int("provider_count", countItems(mbResult)).
|
||||||
|
Int("combined_count", len(combined.Items)).
|
||||||
|
Msg("artist search completed")
|
||||||
|
|
||||||
|
return combined, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MetadataService) deduplicateArtists(dbResult, mbResult *domain.SearchResult[domain.Artist]) *domain.SearchResult[domain.Artist] {
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
var items []domain.Artist
|
||||||
|
total := 0
|
||||||
|
|
||||||
|
addArtist := func(artist domain.Artist) {
|
||||||
|
if artist.Type == "Other" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, ext := range artist.ExternalIDs {
|
||||||
|
key := ext.Source + ":" + ext.SourceID
|
||||||
|
if seen[key] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, ext := range artist.ExternalIDs {
|
||||||
|
seen[ext.Source+":"+ext.SourceID] = true
|
||||||
|
}
|
||||||
|
items = append(items, artist)
|
||||||
|
}
|
||||||
|
|
||||||
|
if dbResult != nil {
|
||||||
|
for _, artist := range dbResult.Items {
|
||||||
|
addArtist(artist)
|
||||||
|
}
|
||||||
|
total = dbResult.Total
|
||||||
|
}
|
||||||
|
|
||||||
|
if mbResult != nil {
|
||||||
|
for _, artist := range mbResult.Items {
|
||||||
|
addArtist(artist)
|
||||||
|
}
|
||||||
|
if mbResult.Total > total {
|
||||||
|
total = mbResult.Total
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
limit := 25
|
||||||
|
offset := 0
|
||||||
|
if dbResult != nil {
|
||||||
|
limit = dbResult.Limit
|
||||||
|
offset = dbResult.Offset
|
||||||
|
} else if mbResult != nil {
|
||||||
|
limit = mbResult.Limit
|
||||||
|
offset = mbResult.Offset
|
||||||
|
}
|
||||||
|
|
||||||
|
return &domain.SearchResult[domain.Artist]{
|
||||||
|
Items: items,
|
||||||
|
Total: total,
|
||||||
|
Limit: limit,
|
||||||
|
Offset: offset,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func countItems[T any](result *domain.SearchResult[T]) int {
|
||||||
|
if result == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return len(result.Items)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MetadataService) SearchAlbums(ctx context.Context, query string, artist string, limit, offset int) (*domain.SearchResult[domain.Album], error) {
|
func (s *MetadataService) SearchAlbums(ctx context.Context, query string, artist string, limit, offset int) (*domain.SearchResult[domain.Album], error) {
|
||||||
|
|||||||
@@ -25,6 +25,10 @@ func (r *noopArtistRepo) Save(ctx context.Context, artist *domain.Artist) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *noopArtistRepo) SaveAll(ctx context.Context, artists []domain.Artist) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type noopAlbumRepo struct{}
|
type noopAlbumRepo struct{}
|
||||||
|
|
||||||
func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {
|
func (r *noopAlbumRepo) GetByID(ctx context.Context, id string) (*domain.Album, error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user