From 025359e765490e519f4a76d9e7621f15c90a1579 Mon Sep 17 00:00:00 2001 From: reckless-sherixx <141176362+reckless-sherixx@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:57:00 +0530 Subject: [PATCH 1/2] parser: don't read a JSON line's key names as its level A container log line has no level from its source, so the line collector guesses one from the raw text. For a JSON line that has no level key, that guess came from the key names: an access log's {"status":200,"error":""} read as an error, because "error" is a key. JSON parsing kept the guess, since it found no level key to override it with. The collector now records that a level was guessed. When the line turns out to be a JSON object without a level key, the parser guesses again from the JSON message field alone ({"msg":"error: db down"} is still an error), or leaves the level unknown. A level the source reports, such as a journald priority, is kept. Both parseJson modes behave the same. --- multiline.go | 29 ++++++++++++++++++----------- parser.go | 12 ++++++++++++ parser_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/multiline.go b/multiline.go index 01dcdde..8f64121 100644 --- a/multiline.go +++ b/multiline.go @@ -20,6 +20,9 @@ type Message struct { // Offset is the largest non-zero Offset of the lines the message consumed, // including lines dropped over the size limit. Offset int64 + // levelGuessed says Level was guessed from the content, not reported by the + // source, so a parser that reads the content as JSON may replace it. + levelGuessed bool } type MultilineCollector struct { @@ -28,12 +31,13 @@ type MultilineCollector struct { timeout time.Duration limit int - ts time.Time - level Level - stream string - lines []string - size int - offset int64 + ts time.Time + level Level + levelGuessed bool + stream string + lines []string + size int + offset int64 lock sync.Mutex closed bool @@ -115,6 +119,7 @@ func (m *MultilineCollector) add(entry LogEntry) { m.level = entry.Level // the level reported by the source is trusted, otherwise it is guessed from the content if m.level == LevelUnknown { m.level = GuessLevel(entry.Content) + m.levelGuessed = true } m.isFirstLineContainsTimestamp = containsTimestamp(entry.Content) } @@ -183,11 +188,12 @@ func (m *MultilineCollector) flushMessage() { } content := strings.TrimSpace(strings.Join(m.lines, "\n")) msg := Message{ - Timestamp: m.ts, - Content: content, - Level: m.level, - Stream: m.stream, - Offset: m.offset, + Timestamp: m.ts, + Content: content, + Level: m.level, + Stream: m.stream, + Offset: m.offset, + levelGuessed: m.levelGuessed, } m.reset() m.Messages <- msg @@ -196,6 +202,7 @@ func (m *MultilineCollector) flushMessage() { func (m *MultilineCollector) reset() { m.ts = time.Time{} m.level = LevelUnknown + m.levelGuessed = false m.stream = "" m.lines = m.lines[:0] m.size = 0 diff --git a/parser.go b/parser.go index 518bced..618d4e3 100644 --- a/parser.go +++ b/parser.go @@ -243,6 +243,18 @@ func (p *Parser) inc(msg Message) { if structuredLevel != LevelUnknown { msg.Level = structuredLevel } + // A JSON line keeps its level in a level key. Without one, a level guessed + // from the raw line came from its key names ({"error":""} reads as an + // error), so it is guessed again from the message alone, or left unknown. + if msg.levelGuessed && structuredLevel == LevelUnknown && (jl == nil || jl.Level == LevelUnknown) { + if jl != nil { + msg.Level = GuessLevel(jl.Message) + } else if !p.parseJson { + if fields := decodeJsonObject(msg.Content); fields != nil { + msg.Level = GuessLevel(jsonLogFromFields(fields).Message) + } + } + } // A JSON log is emitted as its message, with the other fields as // attributes. Patterns still come from normalizedContent. diff --git a/parser_test.go b/parser_test.go index 072811a..38afa9b 100644 --- a/parser_test.go +++ b/parser_test.go @@ -1,6 +1,7 @@ package logparser import ( + "fmt" "sort" "strings" "testing" @@ -269,3 +270,43 @@ func TestParserRateLimitLevels(t *testing.T) { _, ok := p.patterns[patternKey{level: LevelError, hash: sampledPatternHash}] assert.False(t, ok) } + +// A JSON line without a level key gets no level from its key names: "error" in +// {"error":""} is a field, not a severity. A level the source reports is kept. +func TestParserJsonWithoutLevelKey(t *testing.T) { + accessLog := `{"time":"2026-11-03T10:00:00Z","method":"GET","uri":"/healthz","status":200,"error":"","latency_human":"41µs"}` + cases := []struct { + name string + content string + source Level + want Level + }{ + {"error key, no level key", accessLog, LevelUnknown, LevelUnknown}, + {"err key with a value, no level key", `{"method":"POST","status":502,"err":"upstream timeout"}`, LevelUnknown, LevelUnknown}, + {"level guessed from the message", `{"msg":"error: connection refused","attempt":3}`, LevelUnknown, LevelError}, + {"message without a level word", `{"message":"request served","error":""}`, LevelUnknown, LevelUnknown}, + {"level key still wins", `{"level":"info","msg":"done","error":""}`, LevelUnknown, LevelInfo}, + {"source level is kept", accessLog, LevelWarning, LevelWarning}, + {"plain text is still guessed", `ERROR failed to connect to db`, LevelUnknown, LevelError}, + } + for _, parseJson := range []bool{true, false} { + for _, c := range cases { + t.Run(fmt.Sprintf("%s/parseJson=%t", c.name, parseJson), func(t *testing.T) { + levels := make(chan Level, 1) + cb := func(_ time.Time, level Level, _ string, _ string, _ map[string]string, _ string, _ int64) { + levels <- level + } + ch := make(chan LogEntry) + p := NewParser(ch, nil, cb, 10*time.Millisecond, 256, parseJson, nil, SensitiveConfig{}) + defer p.Stop() + ch <- LogEntry{Timestamp: time.Now(), Content: c.content, Level: c.source} + select { + case got := <-levels: + assert.Equal(t, c.want, got) + case <-time.After(5 * time.Second): + t.Fatal("no message emitted") + } + }) + } + } +} From a70b53bb69a7cfd6dc81b98e70d520c8320bf259 Mon Sep 17 00:00:00 2001 From: reckless-sherixx <141176362+reckless-sherixx@users.noreply.github.com> Date: Wed, 18 Nov 2026 02:54:22 +0530 Subject: [PATCH 2/2] test: level word past the raw guess's reach is read from the message --- parser_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parser_test.go b/parser_test.go index 38afa9b..baec411 100644 --- a/parser_test.go +++ b/parser_test.go @@ -284,6 +284,8 @@ func TestParserJsonWithoutLevelKey(t *testing.T) { {"error key, no level key", accessLog, LevelUnknown, LevelUnknown}, {"err key with a value, no level key", `{"method":"POST","status":502,"err":"upstream timeout"}`, LevelUnknown, LevelUnknown}, {"level guessed from the message", `{"msg":"error: connection refused","attempt":3}`, LevelUnknown, LevelError}, + // json.dumps spacing: the raw guess reads only the first fields, so it finds no level word. + {"level word past the raw guess's reach", `{"time": "2026-11-03T10:00:00Z", "logger": "checkout", "pid": 41, "msg": "error: payment declined"}`, LevelUnknown, LevelError}, {"message without a level word", `{"message":"request served","error":""}`, LevelUnknown, LevelUnknown}, {"level key still wins", `{"level":"info","msg":"done","error":""}`, LevelUnknown, LevelInfo}, {"source level is kept", accessLog, LevelWarning, LevelWarning},