diff --git a/.envrc b/.envrc index 3550a30..3036deb 100644 --- a/.envrc +++ b/.envrc @@ -1 +1,8 @@ -use flake +#!/usr/bin/env bash +export GIT_CONFIG_GLOBAL=/dev/null + +eval "$(devenv direnvrc)" + +# You can pass flags to the devenv command +# For example: use devenv --impure --option services.postgres.enable:bool true +use devenv diff --git a/.gitignore b/.gitignore index 0beef97..000b491 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,8 @@ vendor/ docs/research/*/repo/ docs/research/*/repo-index/ + +# Devenv +.devenv* +devenv.local.nix +devenv.local.yaml diff --git a/dev/Metadata Agregator/SearchArtist.yml b/dev/Metadata Agregator/SearchArtist.yml new file mode 100644 index 0000000..376c733 --- /dev/null +++ b/dev/Metadata Agregator/SearchArtist.yml @@ -0,0 +1,17 @@ +info: + name: SearchArtist + type: grpc + seq: 1 + +grpc: + url: localhost:50053 + method: /metadata.v1.MetadataService/SearchArtists + methodType: unary + message: |- + { + "query": "ДДТ", + "limit": 10, + "offset": 0, + "provider": 0 + } + auth: inherit diff --git a/dev/Metadata Agregator/opencollection.yml b/dev/Metadata Agregator/opencollection.yml new file mode 100644 index 0000000..61ae062 --- /dev/null +++ b/dev/Metadata Agregator/opencollection.yml @@ -0,0 +1,25 @@ +opencollection: 1.0.0 + +info: + name: Metadata Agregator +config: + proxy: + inherit: true + config: + protocol: http + hostname: "" + port: "" + auth: + username: "" + password: "" + bypassProxy: "" +bundled: false +extensions: + bruno: + ignore: + - node_modules + - .git + presets: + request: + type: grpc + url: localhost:50053 diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 0000000..1e78357 --- /dev/null +++ b/devenv.lock @@ -0,0 +1,124 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1783461239, + "narHash": "sha256-O+rwg3jreOAOrYd0YrNoOsRu+8is6TCPCIBN3qJTJpM=", + "owner": "cachix", + "repo": "devenv", + "rev": "6222c499fe28b87665af2bb6c626da64179cee00", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "git-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1783008725, + "narHash": "sha256-jGiy6+sxjNWXSjp25uoJuNfyH9zBK1PEDY0lVoL4ibQ=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "bca82caa46d5ec0f5d422c61fb1e30bc51313cbe", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "nixpkgs": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1783345554, + "narHash": "sha256-LZhOm4kqjHUnwP4CNHpaqqEVcumGNd1PvOEOad8LkS0=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "6004ea8c229fe9d41b21c6f4c76bf6c2e10771dd", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs-src": { + "flake": false, + "locked": { + "lastModified": 1783279667, + "narHash": "sha256-/NAkDSsve+GNM0Bt6tleJdCGfsTlK89nPjkVOzZMo0s=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f205b5574fd0cb7da5b702a2da51507b7f4fdd1b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "git-hooks": "git-hooks", + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1780220602, + "narHash": "sha256-eynAfOmbmxJnkp7YewvCEbShNnnYJ9gLLqkzsYtBPeM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "db947814a175b7ca6ded66e21383d938df01c227", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 0000000..c707b9b --- /dev/null +++ b/devenv.nix @@ -0,0 +1,62 @@ +{ + pkgs, + lib, + config, + inputs, + ... +}: +{ + languages.go.enable = true; + + git-hooks.hooks = { + treefmt.enable = true; + }; + + treefmt = { + enable = true; + config.programs = { + nixfmt.enable = true; + gofmt.enable = true; + }; + }; + + packages = with pkgs; [ + pre-commit + gitleaks + plantuml + + gopls + gotools + go-tools + golangci-lint + + buf + protoc-gen-go + protoc-gen-go-grpc + + grpcurl + + opencode + ]; + + services.postgres = { + enable = true; + listen_addresses = "localhost"; + + initialDatabases = [ + { + name = "metadata_agregator"; + schema = ./nix/schema/001_schema.sql; + } + ]; + + initialScript = '' + CREATE USER agregator WITH PASSWORD 'agregator'; + GRANT ALL PRIVILEGES ON DATABASE metadata_agregator TO agregator; + \c metadata_agregator + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO agregator; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO agregator; + GRANT ALL ON SCHEMA public TO agregator; + ''; + }; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 0000000..ee5a179 --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,13 @@ +inputs: + git-hooks: + url: github:cachix/git-hooks.nix + inputs: + nixpkgs: + follows: nixpkgs + nixpkgs: + url: github:cachix/devenv-nixpkgs/rolling + treefmt-nix: + url: github:numtide/treefmt-nix + inputs: + nixpkgs: + follows: nixpkgs diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 1866c51..0000000 --- a/flake.lock +++ /dev/null @@ -1,134 +0,0 @@ -{ - "nodes": { - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1767039857, - "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", - "owner": "NixOS", - "repo": "flake-compat", - "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "lastModified": 1775087534, - "narHash": "sha256-91qqW8lhL7TLwgQWijoGBbiD4t7/q75KTi8NxjVmSmA=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "3107b77cd68437b9a76194f0f7f9c55f2329ca5b", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "git-hooks": { - "inputs": { - "flake-compat": "flake-compat", - "gitignore": "gitignore", - "nixpkgs": "nixpkgs" - }, - "locked": { - "lastModified": 1776796298, - "narHash": "sha256-PcRvlWayisPSjd0UcRQbhG8Oqw78AcPE6x872cPRHN8=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "3cfd774b0a530725a077e17354fbdb87ea1c4aad", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "git-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1709087332, - "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1770073757, - "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "47472570b1e607482890801aeaf29bfb749884f6", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib": { - "locked": { - "lastModified": 1774748309, - "narHash": "sha256-+U7gF3qxzwD5TZuANzZPeJTZRHS29OFQgkQ2kiTJBIQ=", - "owner": "nix-community", - "repo": "nixpkgs.lib", - "rev": "333c4e0545a6da976206c74db8773a1645b5870a", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixpkgs.lib", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1777326013, - "narHash": "sha256-Z+out+ntVqKHsoZdmEM30+84A/and97fv5vouJtev1Q=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "52ffa3b778ab50cc305ff8927aca2b71a0ff50de", - "type": "github" - }, - "original": { - "owner": "nixos", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "flake-parts": "flake-parts", - "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_2" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index eeed5d3..0000000 --- a/flake.nix +++ /dev/null @@ -1,97 +0,0 @@ -{ - inputs = { - nixpkgs.url = "github:nixos/nixpkgs"; - flake-parts.url = "github:hercules-ci/flake-parts"; - git-hooks.url = "github:cachix/git-hooks.nix"; - }; - - outputs = - { - self, - nixpkgs, - flake-parts, - git-hooks, - ... - }@inputs: - flake-parts.lib.mkFlake { inherit inputs; } { - systems = [ - "x86_64-linux" - ]; - - flake = { - nixosModules = { - metadata-agregator = import ./nix/module.nix; - default = self.nixosModules.metadata-agregator; - }; - }; - - perSystem = - { - system, - ... - }: - let - pkgs = import nixpkgs { - inherit system; - config.allowUnfree = true; - }; - - pre-commit-check = git-hooks.lib.${system}.run { - src = ./.; - hooks = { - nixfmt.enable = true; - gofmt.enable = true; - }; - }; - - metadata-server = pkgs.buildGoModule { - pname = "metadata-server"; - version = "0.1.0"; - src = ./.; - vendorHash = "sha256-qNNFHRKqGoo/GveSl9+Bm+DkTKDoSQeGtG8p1edmCJk="; - subPackages = [ "cmd/server" ]; - - meta = { - description = "Music metadata aggregation gRPC service"; - mainProgram = "server"; - }; - }; - in - { - formatter = pkgs.nixfmt-tree; - - packages = { - default = metadata-server; - server = metadata-server; - }; - - checks = { - inherit pre-commit-check; - }; - - devShells.default = pkgs.mkShell { - inherit (pre-commit-check) shellHook; - - buildInputs = with pkgs; [ - pre-commit - gitleaks - plantuml - - go - gopls - gotools - go-tools - golangci-lint - - buf - protoc-gen-go - protoc-gen-go-grpc - - grpcurl - - opencode - ]; - }; - }; - }; -} diff --git a/internal/provider/musicbrainz/client.go b/internal/provider/musicbrainz/client.go index 47b3366..550c0d5 100644 --- a/internal/provider/musicbrainz/client.go +++ b/internal/provider/musicbrainz/client.go @@ -17,8 +17,10 @@ import ( ) const ( - baseURL = "https://musicbrainz.org/ws/2" - userAgent = "MetadataAggregator/0.1.0 (https://github.com/metadata-agregator)" + baseURL = "https://musicbrainz.org/ws/2" + userAgent = "MetadataAggregator/0.1.0 (https://github.com/metadata-agregator)" + maxRetries = 3 + baseBackoff = 2 * time.Second ) type client struct { @@ -36,16 +38,45 @@ func newClient() *client { } func (c *client) get(ctx context.Context, endpoint string, params url.Values) ([]byte, error) { - if err := c.waitForRateLimit(ctx); err != nil { - return nil, err + var lastErr error + + for attempt := 0; attempt <= maxRetries; attempt++ { + if err := c.waitForRateLimit(ctx); err != nil { + return nil, err + } + + req, err := c.buildRequest(ctx, endpoint, params) + if err != nil { + return nil, err + } + + data, err := c.executeAndRead(ctx, req, endpoint) + if err == nil { + return data, nil + } + + if err != ErrRateLimited { + return nil, err + } + + lastErr = err + if attempt < maxRetries { + backoff := baseBackoff * time.Duration(1<