diff --git a/api/traces/v1/trace_rbac.go b/api/traces/v1/trace_rbac.go index 2554446c0..ab38e40f2 100644 --- a/api/traces/v1/trace_rbac.go +++ b/api/traces/v1/trace_rbac.go @@ -85,16 +85,20 @@ func WithTraceQLNamespaceSelectAndForbidOtherAPIs(enabled bool) func(http.Handle } func unmarshal(response *http.Response, body []byte, pb proto.Message) error { - switch response.Header.Get(HeaderContentType) { + contentTypeHeader := response.Header.Get(HeaderContentType) + switch contentTypeHeader { case ContentTypeProtobuf: return proto.Unmarshal(body, pb) - default: + case ContentTypeJSON: return (&jsonpb.Unmarshaler{}).Unmarshal(bytes.NewReader(body), pb) + default: + return fmt.Errorf("unsupported content type: %q", contentTypeHeader) } } func marshal(response *http.Response, buf *bytes.Buffer, pb proto.Message) error { - switch response.Header.Get(HeaderContentType) { + contentTypeHeader := response.Header.Get(HeaderContentType) + switch contentTypeHeader { case ContentTypeProtobuf: b, err := proto.Marshal(pb) if err != nil { @@ -102,8 +106,10 @@ func marshal(response *http.Response, buf *bytes.Buffer, pb proto.Message) error } buf.Write(b) return nil - default: + case ContentTypeJSON: return (&jsonpb.Marshaler{}).Marshal(buf, pb) + default: + return fmt.Errorf("unsupported content type: %q", contentTypeHeader) } } @@ -128,14 +134,18 @@ func responseRBACModifier(log log.Logger) func(response *http.Response) error { responseBuffer := &bytes.Buffer{} switch { case routeQueryV1.MatchString(request.URL.Path): + contentTypeHeader := response.Header.Get(HeaderContentType) + // do not use unmarshal here, because we must use // UnmarshalFromJSONV1 instead of (&jsonpb.Unmarshaler{}).Unmarshal trace := &tempopb.Trace{} - switch response.Header.Get(HeaderContentType) { + switch contentTypeHeader { case ContentTypeProtobuf: err = proto.Unmarshal(b, trace) - default: + case ContentTypeJSON: err = tempopb.UnmarshalFromJSONV1(b, trace) + default: + return fmt.Errorf("unsupported content type: %q", contentTypeHeader) } if err != nil { return err @@ -145,7 +155,7 @@ func responseRBACModifier(log log.Logger) func(response *http.Response) error { // do not use marshal here, because we must use // MarshalToJSONV1 instead of (&jsonpb.Marshaler{}).Marshal - switch response.Header.Get(HeaderContentType) { + switch contentTypeHeader { case ContentTypeProtobuf: var out []byte out, err = proto.Marshal(trace) @@ -153,13 +163,15 @@ func responseRBACModifier(log log.Logger) func(response *http.Response) error { return err } responseBuffer = bytes.NewBuffer(out) - default: + case ContentTypeJSON: var traceResponseBody []byte traceResponseBody, err = tempopb.MarshalToJSONV1(trace) if err != nil { return err } responseBuffer = bytes.NewBuffer(traceResponseBody) + default: + return fmt.Errorf("unsupported content type: %q", contentTypeHeader) } case routeQueryV2.MatchString(request.URL.Path): @@ -209,6 +221,10 @@ func responseRBACModifier(log log.Logger) func(response *http.Response) error { } func traceRBAC(allowedNamespaces map[string]bool, trace *tempopb.Trace) *tempopb.Trace { + if trace == nil { + return nil + } + for _, rs := range trace.ResourceSpans { notAllowedNamespace := "" missingNamespaceAttribute := true diff --git a/api/traces/v1/trace_rbac_test.go b/api/traces/v1/trace_rbac_test.go index a277b5bea..b6fc65cf8 100644 --- a/api/traces/v1/trace_rbac_test.go +++ b/api/traces/v1/trace_rbac_test.go @@ -564,7 +564,7 @@ func contextWithAllowedNamespaces(t *testing.T, namespaces []string) context.Con func makeResponse(ctx context.Context, statusCode int, path string, body string, header http.Header) *http.Response { if header == nil { - header = http.Header{} + header = http.Header{HeaderContentType: []string{ContentTypeJSON}} } return &http.Response{ StatusCode: statusCode, @@ -823,7 +823,7 @@ func TestResponseRBACModifier(t *testing.T) { ] } ] -}`, http.Header{"Content-Encoding": []string{"gzip"}}) +}`, http.Header{"Content-Encoding": []string{"gzip"}, HeaderContentType: []string{ContentTypeJSON}}) require.NoError(t, modifier(resp)) @@ -833,6 +833,64 @@ func TestResponseRBACModifier(t *testing.T) { }) } +func TestMarshalUnmarshalContentType(t *testing.T) { + tests := []struct { + name string + contentType string + wantErr bool + }{ + {"protobuf", ContentTypeProtobuf, false}, + {"json", ContentTypeJSON, false}, + {"unsupported grafana llm json", "application/vnd.grafana.llm+json", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + response := &http.Response{ + Header: http.Header{HeaderContentType: []string{tt.contentType}}, + } + + original := &tempopb.TraceByIDResponse{ + Trace: &tempopb.Trace{ + ResourceSpans: []*tracev1.ResourceSpans{ + { + Resource: &resourcev1.Resource{ + Attributes: []*commonv1.KeyValue{ + createStringAttribute("k8s.namespace.name", "example"), + }, + }, + }, + }, + }, + } + + buf := &bytes.Buffer{} + err := marshal(response, buf, original) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported content type") + assert.Contains(t, err.Error(), tt.contentType) + } else { + require.NoError(t, err) + } + + unmarshalled := &tempopb.TraceByIDResponse{} + err = unmarshal(response, buf.Bytes(), unmarshalled) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported content type") + assert.Contains(t, err.Error(), tt.contentType) + } else { + require.NoError(t, err) + } + + if !tt.wantErr { + assert.Equal(t, original, unmarshalled) + } + }) + } +} + func makeProtobufResponse(ctx context.Context, statusCode int, path string, pb proto.Message) *http.Response { b, err := proto.Marshal(pb) if err != nil { diff --git a/go.mod b/go.mod index 174f9ebaf..8dd8e85b9 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/brancz/kube-rbac-proxy v0.20.1 github.com/coreos/go-oidc/v3 v3.17.0 github.com/efficientgo/core v1.0.0-rc.3.0.20250509094815-bd2ec8febb66 - github.com/efficientgo/e2e v0.14.1-0.20260204162810-8c75b1e33ef9 + github.com/efficientgo/e2e v0.14.1-0.20260927140226-a3483d56ac4b github.com/ghodss/yaml v1.0.0 github.com/go-chi/chi/v5 v5.2.5 github.com/go-chi/httprate v0.15.0 @@ -61,12 +61,14 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dennwc/varint v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.2.1-0.20241212181136-fad1cd13edbd // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect github.com/felixge/httpsnoop v1.1.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logfmt/logfmt v0.6.1 // indirect github.com/go-logr/logr v1.4.4 // indirect @@ -104,6 +106,7 @@ require ( github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/dsig v1.0.0 // indirect github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect @@ -111,19 +114,25 @@ require ( github.com/lestrrat-go/httprc/v3 v3.0.2 // indirect github.com/lestrrat-go/jwx/v3 v3.0.13 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/oklog/ulid/v2 v2.1.1 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tchap/go-patricia/v2 v2.3.3 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/valyala/fastjson v1.6.7 // indirect github.com/vektah/gqlparser/v2 v2.5.31 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/go.sum b/go.sum index 75e3c5c19..c7403e15e 100644 --- a/go.sum +++ b/go.sum @@ -104,8 +104,8 @@ github.com/edsrzf/mmap-go v1.2.1-0.20241212181136-fad1cd13edbd h1:I4PrRZuNMeDP3V github.com/edsrzf/mmap-go v1.2.1-0.20241212181136-fad1cd13edbd/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/efficientgo/core v1.0.0-rc.3.0.20250509094815-bd2ec8febb66 h1:dbyJxOkY0GfW7gCJw2T0XPkxUeal/eKKI24jMJVEd0w= github.com/efficientgo/core v1.0.0-rc.3.0.20250509094815-bd2ec8febb66/go.mod h1:FfGdkzWarkuzOlY04VY+bGfb1lWrjaL6x/GLcQ4vJps= -github.com/efficientgo/e2e v0.14.1-0.20260204162810-8c75b1e33ef9 h1:ubLDovsdOv7n27IV6J+ITPW+VjkbJs/DfAbVqfIFkzQ= -github.com/efficientgo/e2e v0.14.1-0.20260204162810-8c75b1e33ef9/go.mod h1:uQC+38Vj8iASSYXRVkBYXfwUxKiYerKw88G+CH8nsUs= +github.com/efficientgo/e2e v0.14.1-0.20260927140226-a3483d56ac4b h1:TVh1EM/fZodW+VOi+LkW6ptH/PuVdnF2ROW+1NqvtdM= +github.com/efficientgo/e2e v0.14.1-0.20260927140226-a3483d56ac4b/go.mod h1:uQC+38Vj8iASSYXRVkBYXfwUxKiYerKw88G+CH8nsUs= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -267,6 +267,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= diff --git a/ratelimit/http_test.go b/ratelimit/http_test.go index e2d632c22..59dbce6ea 100644 --- a/ratelimit/http_test.go +++ b/ratelimit/http_test.go @@ -433,61 +433,38 @@ func TestWithSharedRateLimiter(t *testing.T) { } } +// launchTestRequests sends reqNum requests to baseURL+pathTest.path one at a time, waiting for +// each response before sending the next. Some callers assert a strict, deterministic progression +// of Retry-After values (see the "shared rate limiter" test cases), which the server assigns in +// the order requests arrive. Sending requests sequentially instead of concurrently keeps launch +// order and arrival order identical, so those assertions aren't flaky under load. func launchTestRequests(t *testing.T, baseURL string, pathTest pathTestParams, reqNum int) (int, int, []http.Header) { - type result struct { - statusCode int - headers http.Header - } - - ordered := make([]result, reqNum) - var wg sync.WaitGroup - var errOnce sync.Once - var requestErr error - - for i := 0; i < reqNum; i++ { - wg.Add(1) - time.Sleep(pathTest.waitBetween) - - go func(i int) { - defer wg.Done() - - res, err := http.Get(baseURL + pathTest.path + "/" + testTenant) - if err != nil { - errOnce.Do(func() { - requestErr = err - }) - return - } - defer res.Body.Close() - - ordered[i] = result{ - statusCode: res.StatusCode, - headers: res.Header.Clone(), - } - }(i) - } - - wg.Wait() - - if requestErr != nil { - t.Fatal(requestErr) - } - var ( gotOKs int gotTooManyRequests int gotHeaders = make([]http.Header, 0, reqNum) ) - for _, r := range ordered { - switch r.statusCode { + for range reqNum { + time.Sleep(pathTest.waitBetween) + + res, err := http.Get(baseURL + pathTest.path + "/" + testTenant) + if err != nil { + t.Fatal(err) + } + + switch res.StatusCode { case http.StatusOK: gotOKs++ case http.StatusTooManyRequests: gotTooManyRequests++ } - gotHeaders = append(gotHeaders, r.headers) + gotHeaders = append(gotHeaders, res.Header.Clone()) + + if err := res.Body.Close(); err != nil { + t.Fatal(err) + } } return gotOKs, gotTooManyRequests, gotHeaders diff --git a/test/e2e/services.go b/test/e2e/services.go index c0e1f5ab6..0f2693764 100644 --- a/test/e2e/services.go +++ b/test/e2e/services.go @@ -35,13 +35,6 @@ const ( gubernatorImage = "ghcr.io/mailgun/gubernator:v2.0.0-rc.36" rulesObjectStoreImage = "quay.io/observatorium/rules-objstore:main-2023-01-05-26de237" - // minioImage contains the Minio image used for testing - // - // This has not been updated in quite a while and Minio has recently changed their licensing terms, which caused - // their public Docker Hub repository to vanish. We now use a Quay mirror, but a better solution would probably - // be to switch to a different S3 service. - minioImage = "quay.io/minio/minio:RELEASE.2022-03-14T18-25-24Z" - logLevelError = "error" logLevelDebug = "debug" ) @@ -68,10 +61,10 @@ func startServicesForMetrics(t *testing.T, e e2e.Environment) ( func startServicesForRules(t *testing.T, e e2e.Environment) (metricsRulesEndpoint string) { // Create S3 replacement for rules backend const bucket = "obs-rules-test" - runnable := e2edb.NewMinio(e, "rules-minio", bucket, e2edb.WithImage(minioImage)) + runnable := e2edb.NewSeaweedFS(e, "rules-storage", bucket) testutil.Ok(t, e2e.StartAndWaitReady(runnable)) - createRulesYAML(t, e, bucket, runnable.InternalEndpoint(e2edb.AccessPortName), e2edb.MinioAccessKey, e2edb.MinioSecretKey) + createRulesYAML(t, e, bucket, runnable.InternalEndpoint(e2edb.AccessPortName), e2edb.S3AccessKey, e2edb.S3SecretKey) rulesBackend := newRulesBackendService(e) testutil.Ok(t, e2e.StartAndWaitReady(rulesBackend)) @@ -83,13 +76,12 @@ func startServicesForLogs(t *testing.T, e e2e.Environment) ( logsEndpoint string, logsExtEndpoint string, ) { - // Create S3 replacement for rules backend - bucket := "loki_test" - runnable := e2edb.NewMinio(e, "loki-minio", bucket, e2edb.WithImage(minioImage)) + bucket := "loki-test" + runnable := e2edb.NewSeaweedFS(e, "loki-storage", bucket) testutil.Ok(t, e2e.StartAndWaitReady(runnable)) - createLokiYAML(t, e, e2edb.MinioAccessKey, e2edb.MinioSecretKey, runnable.InternalEndpoint(e2edb.AccessPortName), bucket) + createLokiYAML(t, e, e2edb.S3AccessKey, e2edb.S3SecretKey, runnable.InternalEndpoint(e2edb.AccessPortName), bucket) loki := newLokiService(e) testutil.Ok(t, e2e.StartAndWaitReady(loki))