diff --git a/internal/ct/ct.go b/internal/ct/ct.go index b287bd5..f56809a 100644 --- a/internal/ct/ct.go +++ b/internal/ct/ct.go @@ -87,6 +87,11 @@ func (c Client) Names(ctx context.Context, domain string) ([]string, error) { // cut instead of losing them all (#251). return sortedNames(set), fmt.Errorf("%w: the answer reached the %d MiB read limit", ErrTruncated, maxBody>>20) } + if body.err != nil && len(set) > 0 { + // The body read failed midway (a timeout, a reset connection): + // keep the names parsed before the cut, as for the limit (#409). + return sortedNames(set), fmt.Errorf("%w: the answer was cut off: %v", ErrTruncated, body.err) + } return nil, fmt.Errorf("certificate transparency lookup: reading %s's answer: %w", resp.Request.URL.Host, err) } return sortedNames(set), nil @@ -124,6 +129,7 @@ type limitReader struct { r io.Reader n int64 hit bool + err error // a read error other than io.EOF, such as a timeout } func (l *limitReader) Read(p []byte) (int, error) { @@ -136,6 +142,9 @@ func (l *limitReader) Read(p []byte) (int, error) { } n, err := l.r.Read(p) l.n -= int64(n) + if err != nil && !errors.Is(err, io.EOF) { + l.err = err + } return n, err } diff --git a/internal/ct/ct_test.go b/internal/ct/ct_test.go index 2e73574..43728b7 100644 --- a/internal/ct/ct_test.go +++ b/internal/ct/ct_test.go @@ -147,6 +147,30 @@ func TestNamesOversizedBody(t *testing.T) { } } +// TestNamesStalledBody covers #409: a body that stalls past the timeout +// keeps the names parsed before it, with ErrTruncated. +func TestNamesStalledBody(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `[{"name_value":"a.example.com"},`) + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer srv.Close() + defer close(release) + + got, err := names(context.Background(), srv, 200*time.Millisecond) + if !errors.Is(err, ErrTruncated) { + t.Errorf("error = %v, want one wrapping ErrTruncated", err) + } + if want := []string{"a"}; !slices.Equal(got, want) { + t.Errorf("Names = %v, want the names before the stall %v", got, want) + } +} + func TestNamesNotAnArray(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"name_value": "a.example.com"}`)) diff --git a/internal/passive/passive_test.go b/internal/passive/passive_test.go index 872a8dc..948b00f 100644 --- a/internal/passive/passive_test.go +++ b/internal/passive/passive_test.go @@ -194,6 +194,28 @@ func TestWaybackTransportError(t *testing.T) { } } +// TestWaybackStalledBody covers #409: a body that stalls past the timeout +// keeps the names read before it, with a Truncated error. +func TestWaybackStalledBody(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("http://a.example.com/\n")) + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer srv.Close() + defer close(release) + + names, err := Wayback{BaseURL: srv.URL, Timeout: 200 * time.Millisecond}.Names(context.Background(), "example.com") + var tr *Truncated + if !errors.As(err, &tr) || tr.Source != "wayback" || !slices.Equal(names, []string{"a"}) { + t.Errorf("names %q, err %v; want [a] and a wayback Truncated error", names, err) + } +} + // FuzzHostNames: any answer parses without panicking, and every name it // gives is a valid entry under the domain (#259). func FuzzHostNames(f *testing.F) { diff --git a/internal/passive/wayback.go b/internal/passive/wayback.go index d241e9f..9c2f538 100644 --- a/internal/passive/wayback.go +++ b/internal/passive/wayback.go @@ -83,6 +83,11 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) { body := &io.LimitedReader{R: resp.Body, N: waybackMaxBody} names, lines, err := hostNames(body, domain) if err != nil { + if len(names) > 0 { + // The body read failed midway (a timeout, a reset connection): + // keep the names read before the cut (#409). + return names, &Truncated{Source: "wayback", Why: fmt.Sprintf("the answer was cut off: %v", err)} + } return nil, fmt.Errorf("wayback machine lookup: reading %s's answer: %w", resp.Request.URL.Host, err) } switch { @@ -99,16 +104,19 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) { const waybackMaxLine = 1 << 20 // hostNames reads one archived URL per line and returns the hosts under -// domain, relative to it, and how many lines it read. +// domain, relative to it, and how many lines it read. On a read error it +// returns the hosts read before it, with the error (#409). func hostNames(r io.Reader, domain string) ([]string, int, error) { domain = strings.ToLower(strings.TrimSuffix(domain, ".")) set := map[string]bool{} br := bufio.NewReaderSize(r, 64<<10) lines := 0 + var readErr error for { raw, tooLong, err := readURLLine(br) if err != nil && !errors.Is(err, io.EOF) { - return nil, lines, err + readErr = err // the partial line before the error is dropped + break } line := strings.TrimSpace(string(raw)) if tooLong { @@ -126,7 +134,7 @@ func hostNames(r io.Reader, domain string) ([]string, int, error) { out = append(out, n) } sort.Strings(out) - return out, lines, nil + return out, lines, readErr } // readURLLine reads one line, without its newline. A line over