Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions internal/ct/ct.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ func (c Client) Names(ctx context.Context, domain string) ([]string, error) {
// cut instead of losing them all (#251).
return sortedNames(set), fmt.Errorf("%w: the answer reached the %d MiB read limit", ErrTruncated, maxBody>>20)
}
if body.err != nil && len(set) > 0 {
// The body read failed midway (a timeout, a reset connection):
// keep the names parsed before the cut, as for the limit (#409).
return sortedNames(set), fmt.Errorf("%w: the answer was cut off: %v", ErrTruncated, body.err)
}
return nil, fmt.Errorf("certificate transparency lookup: reading %s's answer: %w", resp.Request.URL.Host, err)
}
return sortedNames(set), nil
Expand Down Expand Up @@ -124,6 +129,7 @@ type limitReader struct {
r io.Reader
n int64
hit bool
err error // a read error other than io.EOF, such as a timeout
}

func (l *limitReader) Read(p []byte) (int, error) {
Expand All @@ -136,6 +142,9 @@ func (l *limitReader) Read(p []byte) (int, error) {
}
n, err := l.r.Read(p)
l.n -= int64(n)
if err != nil && !errors.Is(err, io.EOF) {
l.err = err
}
return n, err
}

Expand Down
24 changes: 24 additions & 0 deletions internal/ct/ct_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,30 @@ func TestNamesOversizedBody(t *testing.T) {
}
}

// TestNamesStalledBody covers #409: a body that stalls past the timeout
// keeps the names parsed before it, with ErrTruncated.
func TestNamesStalledBody(t *testing.T) {
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = io.WriteString(w, `[{"name_value":"a.example.com"},`)
w.(http.Flusher).Flush()
select {
case <-r.Context().Done():
case <-release:
}
}))
defer srv.Close()
defer close(release)

got, err := names(context.Background(), srv, 200*time.Millisecond)
if !errors.Is(err, ErrTruncated) {
t.Errorf("error = %v, want one wrapping ErrTruncated", err)
}
if want := []string{"a"}; !slices.Equal(got, want) {
t.Errorf("Names = %v, want the names before the stall %v", got, want)
}
}

func TestNamesNotAnArray(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"name_value": "a.example.com"}`))
Expand Down
22 changes: 22 additions & 0 deletions internal/passive/passive_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,28 @@ func TestWaybackTransportError(t *testing.T) {
}
}

// TestWaybackStalledBody covers #409: a body that stalls past the timeout
// keeps the names read before it, with a Truncated error.
func TestWaybackStalledBody(t *testing.T) {
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("http://a.example.com/\n"))
w.(http.Flusher).Flush()
select {
case <-r.Context().Done():
case <-release:
}
}))
defer srv.Close()
defer close(release)

names, err := Wayback{BaseURL: srv.URL, Timeout: 200 * time.Millisecond}.Names(context.Background(), "example.com")
var tr *Truncated
if !errors.As(err, &tr) || tr.Source != "wayback" || !slices.Equal(names, []string{"a"}) {
t.Errorf("names %q, err %v; want [a] and a wayback Truncated error", names, err)
}
}

// FuzzHostNames: any answer parses without panicking, and every name it
// gives is a valid entry under the domain (#259).
func FuzzHostNames(f *testing.F) {
Expand Down
14 changes: 11 additions & 3 deletions internal/passive/wayback.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) {
body := &io.LimitedReader{R: resp.Body, N: waybackMaxBody}
names, lines, err := hostNames(body, domain)
if err != nil {
if len(names) > 0 {
// The body read failed midway (a timeout, a reset connection):
// keep the names read before the cut (#409).
return names, &Truncated{Source: "wayback", Why: fmt.Sprintf("the answer was cut off: %v", err)}
}
return nil, fmt.Errorf("wayback machine lookup: reading %s's answer: %w", resp.Request.URL.Host, err)
}
switch {
Expand All @@ -99,16 +104,19 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) {
const waybackMaxLine = 1 << 20

// hostNames reads one archived URL per line and returns the hosts under
// domain, relative to it, and how many lines it read.
// domain, relative to it, and how many lines it read. On a read error it
// returns the hosts read before it, with the error (#409).
func hostNames(r io.Reader, domain string) ([]string, int, error) {
domain = strings.ToLower(strings.TrimSuffix(domain, "."))
set := map[string]bool{}
br := bufio.NewReaderSize(r, 64<<10)
lines := 0
var readErr error
for {
raw, tooLong, err := readURLLine(br)
if err != nil && !errors.Is(err, io.EOF) {
return nil, lines, err
readErr = err // the partial line before the error is dropped
break
}
line := strings.TrimSpace(string(raw))
if tooLong {
Expand All @@ -126,7 +134,7 @@ func hostNames(r io.Reader, domain string) ([]string, int, error) {
out = append(out, n)
}
sort.Strings(out)
return out, lines, nil
return out, lines, readErr
}

// readURLLine reads one line, without its newline. A line over
Expand Down