feat: fuzzy search with parallel DB/MusicBrainz execution
- Add pg_trgm fuzzy search to PostgreSQL artist repository - Add Lucene fuzzy query (~0.7) for MusicBrainz artist search - Run DB and MusicBrainz searches in parallel goroutines - Cache MusicBrainz results to DB for accumulation - Deduplicate results by external ID - Filter out special purpose artists (type=Other) - Add SaveAll method for batch artist inserts - Move database schema to containers repo
This commit is contained in:
Alexander
@@ -60,23 +60,31 @@ func (r *ArtistRepository) GetByExternalID(ctx context.Context, source, sourceID
|
||||
}
|
||||
|
||||
func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offset int) (*domain.SearchResult[domain.Artist], error) {
|
||||
countQuery := `SELECT COUNT(*) FROM artists WHERE name ILIKE $1`
|
||||
// Use pg_trgm similarity for fuzzy matching with a threshold of 0.3
|
||||
// Combined with ILIKE for substring matches
|
||||
countQuery := `
|
||||
SELECT COUNT(*) FROM artists
|
||||
WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3`
|
||||
searchQuery := `
|
||||
SELECT id, name, sort_name, artist_type, country, formed_date, disbanded_date,
|
||||
description, image_url, source, source_id
|
||||
description, image_url, source, source_id,
|
||||
GREATEST(
|
||||
similarity(lower(name), lower($2)),
|
||||
CASE WHEN name ILIKE $1 THEN 0.5 ELSE 0 END
|
||||
) as score
|
||||
FROM artists
|
||||
WHERE name ILIKE $1
|
||||
ORDER BY name
|
||||
LIMIT $2 OFFSET $3`
|
||||
WHERE name ILIKE $1 OR similarity(lower(name), lower($2)) > 0.3
|
||||
ORDER BY score DESC, name
|
||||
LIMIT $3 OFFSET $4`
|
||||
|
||||
pattern := "%" + query + "%"
|
||||
|
||||
var total int
|
||||
if err := r.pool.QueryRow(ctx, countQuery, pattern).Scan(&total); err != nil {
|
||||
if err := r.pool.QueryRow(ctx, countQuery, pattern, query).Scan(&total); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := r.pool.Query(ctx, searchQuery, pattern, limit, offset)
|
||||
rows, err := r.pool.Query(ctx, searchQuery, pattern, query, limit, offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -84,7 +92,7 @@ func (r *ArtistRepository) Search(ctx context.Context, query string, limit, offs
|
||||
|
||||
var artists []domain.Artist
|
||||
for rows.Next() {
|
||||
artist, err := r.scanArtistFromRow(rows)
|
||||
artist, err := r.scanArtistFromRowWithScore(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -152,20 +160,79 @@ func (r *ArtistRepository) Save(ctx context.Context, artist *domain.Artist) erro
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (r *ArtistRepository) SaveAll(ctx context.Context, artists []domain.Artist) error {
|
||||
if len(artists) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
for _, artist := range artists {
|
||||
var source, sourceID string
|
||||
if len(artist.ExternalIDs) > 0 {
|
||||
source = artist.ExternalIDs[0].Source
|
||||
sourceID = artist.ExternalIDs[0].SourceID
|
||||
}
|
||||
|
||||
query := `
|
||||
INSERT INTO artists (id, name, sort_name, artist_type, country, formed_date,
|
||||
disbanded_date, description, image_url, source, source_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
sort_name = EXCLUDED.sort_name,
|
||||
artist_type = EXCLUDED.artist_type,
|
||||
country = EXCLUDED.country,
|
||||
formed_date = EXCLUDED.formed_date,
|
||||
disbanded_date = EXCLUDED.disbanded_date,
|
||||
description = EXCLUDED.description,
|
||||
image_url = EXCLUDED.image_url,
|
||||
updated_at = now()`
|
||||
|
||||
_, err = tx.Exec(ctx, query,
|
||||
artist.ID, artist.Name, nullString(artist.SortName), nullString(artist.Type),
|
||||
nullString(artist.Country), artist.FormedDate, artist.DisbandedDate,
|
||||
nullString(artist.Description), nullString(artist.ImageURL), source, sourceID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, ext := range artist.ExternalIDs {
|
||||
extQuery := `
|
||||
INSERT INTO artist_external_ids (artist_id, source, source_id, url)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (artist_id, source, source_id) DO UPDATE SET
|
||||
url = EXCLUDED.url,
|
||||
fetched_at = now()`
|
||||
|
||||
_, err = tx.Exec(ctx, extQuery, artist.ID, ext.Source, ext.SourceID, nullString(ext.URL))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ...any) (*domain.Artist, error) {
|
||||
row := r.pool.QueryRow(ctx, query, args...)
|
||||
|
||||
var (
|
||||
artist domain.Artist
|
||||
sortName *string
|
||||
artistType *string
|
||||
country *string
|
||||
formedDate *time.Time
|
||||
disbandDate *time.Time
|
||||
description *string
|
||||
imageURL *string
|
||||
source string
|
||||
sourceID *string
|
||||
artist domain.Artist
|
||||
sortName *string
|
||||
artistType *string
|
||||
country *string
|
||||
formedDate *time.Time
|
||||
disbandDate *time.Time
|
||||
description *string
|
||||
imageURL *string
|
||||
source string
|
||||
sourceID *string
|
||||
)
|
||||
|
||||
err := row.Scan(
|
||||
@@ -192,16 +259,16 @@ func (r *ArtistRepository) scanArtist(ctx context.Context, query string, args ..
|
||||
|
||||
func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error) {
|
||||
var (
|
||||
artist domain.Artist
|
||||
sortName *string
|
||||
artistType *string
|
||||
country *string
|
||||
formedDate *time.Time
|
||||
disbandDate *time.Time
|
||||
description *string
|
||||
imageURL *string
|
||||
source string
|
||||
sourceID *string
|
||||
artist domain.Artist
|
||||
sortName *string
|
||||
artistType *string
|
||||
country *string
|
||||
formedDate *time.Time
|
||||
disbandDate *time.Time
|
||||
description *string
|
||||
imageURL *string
|
||||
source string
|
||||
sourceID *string
|
||||
)
|
||||
|
||||
err := row.Scan(
|
||||
@@ -223,6 +290,40 @@ func (r *ArtistRepository) scanArtistFromRow(row pgx.Row) (*domain.Artist, error
|
||||
return &artist, nil
|
||||
}
|
||||
|
||||
func (r *ArtistRepository) scanArtistFromRowWithScore(row pgx.Row) (*domain.Artist, error) {
|
||||
var (
|
||||
artist domain.Artist
|
||||
sortName *string
|
||||
artistType *string
|
||||
country *string
|
||||
formedDate *time.Time
|
||||
disbandDate *time.Time
|
||||
description *string
|
||||
imageURL *string
|
||||
source string
|
||||
sourceID *string
|
||||
score float64
|
||||
)
|
||||
|
||||
err := row.Scan(
|
||||
&artist.ID, &artist.Name, &sortName, &artistType, &country,
|
||||
&formedDate, &disbandDate, &description, &imageURL, &source, &sourceID, &score,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
artist.SortName = derefString(sortName)
|
||||
artist.Type = derefString(artistType)
|
||||
artist.Country = derefString(country)
|
||||
artist.FormedDate = formedDate
|
||||
artist.DisbandedDate = disbandDate
|
||||
artist.Description = derefString(description)
|
||||
artist.ImageURL = derefString(imageURL)
|
||||
|
||||
return &artist, nil
|
||||
}
|
||||
|
||||
func (r *ArtistRepository) loadExternalIDs(ctx context.Context, artist *domain.Artist) error {
|
||||
query := `SELECT source, source_id, url FROM artist_external_ids WHERE artist_id = $1`
|
||||
|
||||
|
||||
Reference in New Issue
Block a user