diff --git a/internal/handler/conan.go b/internal/handler/conan.go index aa41668b..d4040e20 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -1,7 +1,6 @@ package handler import ( - "fmt" "net/http" "strings" ) @@ -41,13 +40,12 @@ func (h *ConanHandler) Routes() http.Handler { mux.HandleFunc("GET /v1/ping", h.handlePing) mux.HandleFunc("GET /v2/ping", h.handlePing) - // Recipe file downloads (cache these) - mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile) - mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/recipe/{filename}", h.handleRecipeFile) - - // Package file downloads (cache these) - mux.HandleFunc("GET /v1/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile) - mux.HandleFunc("GET /v2/files/{name}/{version}/{user}/{channel}/{revision}/package/{pkgref}/{pkgrev}/{filename}", h.handlePackageFile) + // Recipe and package file downloads, as built by the Conan 2 client + // (ClientV2Router.recipe_file and package_file). The v1 protocol hands out + // signed absolute URLs on the upstream host, so its downloads never pass + // through here. + mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/files/{filename}", h.handleRecipeFile) + mux.HandleFunc("GET /v2/conans/{name}/{version}/{user}/{channel}/revisions/{revision}/packages/{pkgref}/revisions/{pkgrev}/files/{filename}", h.handlePackageFile) // Proxy all other endpoints (metadata, search, etc.) with caching mux.HandleFunc("GET /", h.proxyCached) @@ -70,25 +68,19 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) revision := r.PathValue("revision") filename := r.PathValue("filename") - // Only cache specific files + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { - h.proxyUpstream(w, r) + h.proxyCached(w, r) return } - // Conan package name format: name/version@user/channel - packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel) - h.proxy.Logger.Info("conan recipe download", "name", name, "version", version, "user", user, "channel", channel, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path + storageFilename := conanStorageFilename(user, channel, revision+"_"+filename) - // Use revision as part of version for storage - storageVersion := fmt.Sprintf("%s_%s", version, revision) - storageFilename := fmt.Sprintf("recipe_%s", filename) - - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch file") return @@ -108,24 +100,19 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) pkgrev := r.PathValue("pkgrev") filename := r.PathValue("filename") - // Only cache specific files + // Only the archives are artifacts; the rest goes through the metadata cache. if !h.shouldCacheFile(filename) { - h.proxyUpstream(w, r) + h.proxyCached(w, r) return } - packageName := fmt.Sprintf("%s/%s@%s/%s", name, version, user, channel) - h.proxy.Logger.Info("conan package download", "name", name, "version", version, "pkgref", pkgref, "filename", filename) upstreamURL := h.upstreamURL + r.URL.Path + storageFilename := conanStorageFilename(user, channel, revision+"_"+pkgref+"_"+pkgrev+"_"+filename) - // Use revision and package ref as part of version for storage - storageVersion := fmt.Sprintf("%s_%s_%s_%s", version, revision, pkgref, pkgrev) - storageFilename := fmt.Sprintf("package_%s", filename) - - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", packageName, storageVersion, storageFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conan", name, version, storageFilename, upstreamURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch file") return @@ -134,13 +121,24 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) ServeArtifactRequest(w, r, result) } +// conanStorageFilename names a cached archive. Archives are stored under +// pkg:conan/{name}@{version}, so the file name carries the revisions and, when +// set, user and channel, to keep every recipe and binary apart. +func conanStorageFilename(user, channel, file string) string { + if user == "_" && channel == "_" { + return file + } + return user + "@" + channel + "_" + file +} + // shouldCacheFile returns true if the file should be cached. func (h *ConanHandler) shouldCacheFile(filename string) bool { - // Cache the large archive files + // Cache the large archive files. Since Conan 2.25 they can also be xz or + // zstd (core.upload:compression_format). cacheFiles := []string{ - "conan_sources.tgz", - "conan_export.tgz", - "conan_package.tgz", + "conan_sources.tgz", "conan_sources.txz", "conan_sources.tzst", + "conan_export.tgz", "conan_export.txz", "conan_export.tzst", + "conan_package.tgz", "conan_package.txz", "conan_package.tzst", } for _, f := range cacheFiles { diff --git a/internal/handler/conan_test.go b/internal/handler/conan_test.go index a7bd3628..f2fb909f 100644 --- a/internal/handler/conan_test.go +++ b/internal/handler/conan_test.go @@ -6,7 +6,9 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" + "time" ) const testProxyURL = "http://localhost:8080" @@ -28,6 +30,13 @@ func TestConanShouldCacheFile(t *testing.T) { {"conan_sources.tgz", true}, {"conan_export.tgz", true}, {"conan_package.tgz", true}, + {"conan_sources.txz", true}, + {"conan_export.txz", true}, + {"conan_package.txz", true}, + {"conan_sources.tzst", true}, + {"conan_export.tzst", true}, + {"conan_package.tzst", true}, + {"conan_package.tar.gz", false}, {"conanfile.py", false}, {"conanmanifest.txt", false}, {"conaninfo.txt", false}, @@ -225,7 +234,7 @@ func TestConanRecipeFileNonCacheable(t *testing.T) { proxyURL: "http://proxy.local", } - req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/recipe/conanfile.py", nil) + req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/files/conanfile.py", nil) req.SetPathValue("name", "zlib") req.SetPathValue("version", "1.2.13") req.SetPathValue("user", "_") @@ -260,7 +269,7 @@ func TestConanPackageFileNonCacheable(t *testing.T) { proxyURL: "http://proxy.local", } - req := httptest.NewRequest(http.MethodGet, "/v2/files/zlib/1.2.13/_/_/abc123/package/pkgref1/pkgrev1/conaninfo.txt", nil) + req := httptest.NewRequest(http.MethodGet, "/v2/conans/zlib/1.2.13/_/_/revisions/abc123/packages/pkgref1/revisions/pkgrev1/files/conaninfo.txt", nil) req.SetPathValue("name", "zlib") req.SetPathValue("version", "1.2.13") req.SetPathValue("user", "_") @@ -283,6 +292,173 @@ func TestConanPackageFileNonCacheable(t *testing.T) { } } +// The Conan 2 client downloads recipe and package files over these routes; +// the archives must be stored as artifacts and served from the cache after. +func TestConanV2FileRoutesCacheArchives(t *testing.T) { + tests := []struct { + name string + path string + filename string + }{ + {"recipe", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", + "rrev1_conan_sources.tgz"}, + {"package", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", + "rrev1_pkgid1_prev1_conan_package.tgz"}, + {"recipe xz", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_export.txz", + "rrev1_conan_export.txz"}, + {"package zstd", "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tzst", + "rrev1_pkgid1_prev1_conan_package.tzst"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("archive %s was proxied instead of fetched as an artifact", r.URL.Path) + http.Error(w, "unexpected", http.StatusInternalServerError) + })) + defer upstream.Close() + proxy.HTTPClient = upstream.Client() + + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for i := range 2 { + fetcher.artifact = artifactBody("archive bytes") + fetcher.fetchCalled = false + + resp, err := http.Get(srv.URL + tt.path) + if err != nil { + t.Fatalf("request %d failed: %v", i+1, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusOK || string(body) != "archive bytes" { + t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "archive bytes") + } + if fetcher.fetchCalled != (i == 0) { + t.Errorf("request %d: fetched = %v, want %v", i+1, fetcher.fetchCalled, i == 0) + } + if i == 0 && fetcher.fetchedURL != upstream.URL+tt.path { + t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, upstream.URL+tt.path) + } + } + + recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", tt.filename) + }) + } +} + +// References that differ only in user and channel share pkg:conan/zlib@1.3.1 +// but must not share files, even with the same recipe revision. +func TestConanUserChannelKeptApart(t *testing.T) { + proxy, db, _, fetcher := setupTestProxy(t) + h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for _, ref := range []struct{ path, body, filename string }{ + {"/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", + "center sources", "rrev1_conan_sources.tgz"}, + {"/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/files/conan_sources.tgz", + "acme sources", "acme@stable_rrev1_conan_sources.tgz"}, + } { + fetcher.artifact = artifactBody(ref.body) + fetcher.fetchCalled = false + + resp, err := http.Get(srv.URL + ref.path) + if err != nil { + t.Fatalf("GET %s failed: %v", ref.path, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if string(body) != ref.body || !fetcher.fetchCalled { + t.Errorf("GET %s: body = %q, fetched = %v, want %q from upstream", ref.path, body, fetcher.fetchCalled, ref.body) + } + recordedStoragePath(t, db, "pkg:conan/zlib@1.3.1", ref.filename) + } +} + +// A denylist entry for a Conan version blocks its recipe and binary archives. +func TestConanDenylistMatchesVersion(t *testing.T) { + proxy, _, _, fetcher := setupTestProxy(t) + setTestDenylist(t, proxy, "pkg:conan/zlib@1.3.1") + h := NewConanHandlerWithUpstream(proxy, testProxyURL, "http://upstream.invalid") + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for _, path := range []string{ + "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conan_sources.tgz", + "/v2/conans/zlib/1.3.1/acme/stable/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conan_package.tgz", + } { + fetcher.artifact = artifactBody("archive bytes") + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("GET %s failed: %v", path, err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusForbidden { + t.Errorf("GET %s: status = %d, want %d", path, resp.StatusCode, http.StatusForbidden) + } + } + if fetcher.fetchCalled { + t.Error("denied archive was fetched from upstream") + } +} + +// The small files next to the archives keep going through the metadata cache. +func TestConanV2FileRoutesKeepSmallFilesAsMetadata(t *testing.T) { + for name, path := range map[string]string{ + "recipe": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/files/conanmanifest.txt", + "package": "/v2/conans/zlib/1.3.1/_/_/revisions/rrev1/packages/pkgid1/revisions/prev1/files/conaninfo.txt", + } { + t.Run(name, func(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Path != path { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("small file")) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + + h := NewConanHandlerWithUpstream(proxy, testProxyURL, upstream.URL) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + for i := range 2 { + resp, err := http.Get(srv.URL + path) + if err != nil { + t.Fatalf("request %d failed: %v", i+1, err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || string(body) != "small file" { + t.Fatalf("request %d: status = %d, body = %q, want 200 %q", i+1, resp.StatusCode, body, "small file") + } + } + + if got := requests.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (second served from the metadata cache)", got) + } + if fetcher.fetchCalled { + t.Error("small file was fetched as an artifact") + } + }) + } +} + func TestConanRoutes(t *testing.T) { h := &ConanHandler{ proxy: conanTestProxy(), diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index d91b68ad..0df443d7 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1163,7 +1163,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz") + resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz") if err != nil { t.Fatalf("request failed: %v", err) } @@ -1173,7 +1173,7 @@ func TestConanHandler_RecipeFileCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } - want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/recipe/conan_export.tgz" + want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/files/conan_export.tgz" if fetcher.fetchedURL != want { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } @@ -1190,7 +1190,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) { srv := httptest.NewServer(h.Routes()) defer srv.Close() - resp, err := http.Get(srv.URL + "/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz") + resp, err := http.Get(srv.URL + "/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz") if err != nil { t.Fatalf("request failed: %v", err) } @@ -1200,7 +1200,7 @@ func TestConanHandler_PackageFileCacheMiss(t *testing.T) { t.Error("expected fetcher to be called on cache miss") } - want := "https://center.conan.io/v2/files/zlib/1.3/_/_/abc123/package/def456/ghi789/conan_package.tgz" + want := "https://center.conan.io/v2/conans/zlib/1.3/_/_/revisions/abc123/packages/def456/revisions/ghi789/files/conan_package.tgz" if fetcher.fetchedURL != want { t.Errorf("upstream URL = %q, want %q", fetcher.fetchedURL, want) } diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index 22cc3d29..9e229fcc 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -31,7 +31,7 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { func(p *Proxy) http.Handler { return NewCRANHandler(p, "http://localhost").Routes() }}, {"conda", "/conda-forge/linux-64/numpy-1.26.0-py311_0.tar.bz2", func(p *Proxy) http.Handler { return NewCondaHandler(p, "http://localhost").Routes() }}, - {"conan", "/v1/files/zlib/1.3.1/_/_/0/recipe/conan_sources.tgz", + {"conan", "/v2/conans/zlib/1.3.1/_/_/revisions/0/files/conan_sources.tgz", func(p *Proxy) http.Handler { return NewConanHandler(p, "http://localhost").Routes() }}, {"gem", "/gems/rails-7.1.0.gem", func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go index 67af121c..a7a8fb79 100644 --- a/internal/handler/relay_test.go +++ b/internal/handler/relay_test.go @@ -50,7 +50,7 @@ func relayTestRoutes(proxy *Proxy, upstream string) http.Handler { func TestRelayRoutes(t *testing.T) { for _, route := range []string{ "/upstream", "/file", "/metadata", "/nuget/query", - "/conan/v2/files/demo/1.0/user/stable/rev/recipe/other.txt", + "/conan/v2/conans/demo/1.0/user/stable/revisions/rev/files/other.txt", "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", "/gem/info/demo", "/conda/conda-forge/noarch/repodata.json", "/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift",