Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions multiline.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ type Message struct {
// Offset is the largest non-zero Offset of the lines the message consumed,
// including lines dropped over the size limit.
Offset int64
// levelGuessed says Level was guessed from the content, not reported by the
// source, so a parser that reads the content as JSON may replace it.
levelGuessed bool
}

type MultilineCollector struct {
Expand All @@ -28,12 +31,13 @@ type MultilineCollector struct {
timeout time.Duration
limit int

ts time.Time
level Level
stream string
lines []string
size int
offset int64
ts time.Time
level Level
levelGuessed bool
stream string
lines []string
size int
offset int64

lock sync.Mutex
closed bool
Expand Down Expand Up @@ -115,6 +119,7 @@ func (m *MultilineCollector) add(entry LogEntry) {
m.level = entry.Level // the level reported by the source is trusted, otherwise it is guessed from the content
if m.level == LevelUnknown {
m.level = GuessLevel(entry.Content)
m.levelGuessed = true
}
m.isFirstLineContainsTimestamp = containsTimestamp(entry.Content)
}
Expand Down Expand Up @@ -183,11 +188,12 @@ func (m *MultilineCollector) flushMessage() {
}
content := strings.TrimSpace(strings.Join(m.lines, "\n"))
msg := Message{
Timestamp: m.ts,
Content: content,
Level: m.level,
Stream: m.stream,
Offset: m.offset,
Timestamp: m.ts,
Content: content,
Level: m.level,
Stream: m.stream,
Offset: m.offset,
levelGuessed: m.levelGuessed,
}
m.reset()
m.Messages <- msg
Expand All @@ -196,6 +202,7 @@ func (m *MultilineCollector) flushMessage() {
func (m *MultilineCollector) reset() {
m.ts = time.Time{}
m.level = LevelUnknown
m.levelGuessed = false
m.stream = ""
m.lines = m.lines[:0]
m.size = 0
Expand Down
12 changes: 12 additions & 0 deletions parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,18 @@ func (p *Parser) inc(msg Message) {
if structuredLevel != LevelUnknown {
msg.Level = structuredLevel
}
// A JSON line keeps its level in a level key. Without one, a level guessed
// from the raw line came from its key names ({"error":""} reads as an
// error), so it is guessed again from the message alone, or left unknown.
if msg.levelGuessed && structuredLevel == LevelUnknown && (jl == nil || jl.Level == LevelUnknown) {
if jl != nil {
msg.Level = GuessLevel(jl.Message)
} else if !p.parseJson {
if fields := decodeJsonObject(msg.Content); fields != nil {
msg.Level = GuessLevel(jsonLogFromFields(fields).Message)
}
}
}
Comment on lines +249 to +257

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

When p.parseJson is false, the parser treats logs as plain text but attempts to decode JSON to correct potential false-positive level guesses (e.g., from an "error" key name). However, if msg.Level is already LevelUnknown, there is no guessed level to correct.

By adding msg.Level != LevelUnknown to the condition, we can completely avoid the expensive decodeJsonObject call (which uses json.NewDecoder and allocates maps) for all log lines that are already LevelUnknown. This is a significant performance optimization for high-throughput log streams when JSON parsing is disabled.

Suggested change
if msg.levelGuessed && structuredLevel == LevelUnknown && (jl == nil || jl.Level == LevelUnknown) {
if jl != nil {
msg.Level = GuessLevel(jl.Message)
} else if !p.parseJson {
if fields := decodeJsonObject(msg.Content); fields != nil {
msg.Level = GuessLevel(jsonLogFromFields(fields).Message)
}
}
}
if msg.levelGuessed && structuredLevel == LevelUnknown && (jl == nil || jl.Level == LevelUnknown) {
if jl != nil {
msg.Level = GuessLevel(jl.Message)
} else if !p.parseJson && msg.Level != LevelUnknown {
if fields := decodeJsonObject(msg.Content); fields != nil {
msg.Level = GuessLevel(jsonLogFromFields(fields).Message)
}
}
}

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not applying this one: it changes which lines get a level, not just how fast.

msg.Level being unknown here doesn't mean the message has no level word. The raw-line guess only reads the first 7 whitespace-separated fields (and 255 bytes), so it misses a level word that sits later in the line. JSON written with json.dumps spacing is the common case:

{"time": "2026-11-03T10:00:00Z", "logger": "checkout", "pid": 41, "msg": "error: payment declined"}

The raw guess is unknown; the message guess is error. With the suggested msg.Level != LevelUnknown guard, this line comes out error with parseJson on and unknown with it off. I checked this with a test case added to TestParserJsonWithoutLevelKey, which fails under the guard.

On cost: decodeJsonObject returns before allocating for anything that doesn't start with { and end with }. So the decode only runs for JSON-shaped lines that have no level key, and only when parseJson is off.


// A JSON log is emitted as its message, with the other fields as
// attributes. Patterns still come from normalizedContent.
Expand Down
43 changes: 43 additions & 0 deletions parser_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package logparser

import (
"fmt"
"sort"
"strings"
"testing"
Expand Down Expand Up @@ -269,3 +270,45 @@ func TestParserRateLimitLevels(t *testing.T) {
_, ok := p.patterns[patternKey{level: LevelError, hash: sampledPatternHash}]
assert.False(t, ok)
}

// A JSON line without a level key gets no level from its key names: "error" in
// {"error":""} is a field, not a severity. A level the source reports is kept.
func TestParserJsonWithoutLevelKey(t *testing.T) {
accessLog := `{"time":"2026-11-03T10:00:00Z","method":"GET","uri":"/healthz","status":200,"error":"","latency_human":"41µs"}`
cases := []struct {
name string
content string
source Level
want Level
}{
{"error key, no level key", accessLog, LevelUnknown, LevelUnknown},
{"err key with a value, no level key", `{"method":"POST","status":502,"err":"upstream timeout"}`, LevelUnknown, LevelUnknown},
{"level guessed from the message", `{"msg":"error: connection refused","attempt":3}`, LevelUnknown, LevelError},
// json.dumps spacing: the raw guess reads only the first fields, so it finds no level word.
{"level word past the raw guess's reach", `{"time": "2026-11-03T10:00:00Z", "logger": "checkout", "pid": 41, "msg": "error: payment declined"}`, LevelUnknown, LevelError},
{"message without a level word", `{"message":"request served","error":""}`, LevelUnknown, LevelUnknown},
{"level key still wins", `{"level":"info","msg":"done","error":""}`, LevelUnknown, LevelInfo},
{"source level is kept", accessLog, LevelWarning, LevelWarning},
{"plain text is still guessed", `ERROR failed to connect to db`, LevelUnknown, LevelError},
}
for _, parseJson := range []bool{true, false} {
for _, c := range cases {
t.Run(fmt.Sprintf("%s/parseJson=%t", c.name, parseJson), func(t *testing.T) {
levels := make(chan Level, 1)
cb := func(_ time.Time, level Level, _ string, _ string, _ map[string]string, _ string, _ int64) {
levels <- level
}
ch := make(chan LogEntry)
p := NewParser(ch, nil, cb, 10*time.Millisecond, 256, parseJson, nil, SensitiveConfig{})
defer p.Stop()
ch <- LogEntry{Timestamp: time.Now(), Content: c.content, Level: c.source}
select {
case got := <-levels:
assert.Equal(t, c.want, got)
case <-time.After(5 * time.Second):
t.Fatal("no message emitted")
}
})
}
}
}
Loading