app/vlinsert/loki: parse JSON message with trailing whitespace/newlines (#1045)

Signed-off-by: Aliaksandr Valialkin <valyala@victoriametrics.com>
Co-authored-by: Aliaksandr Valialkin <valyala@victoriametrics.com>

Author: func25

app/vlinsert/loki/loki_json.go

@@ -182,20 +182,44 @@ func parseJSONRequest(data []byte, lmp insertutil.LogMessageProcessor, msgFields
 }
 
 func addMsgField(fs *logstorage.Fields, msgParser *logstorage.JSONParser, msg string, preserveKeys []string) bool {
-	if msgParser == nil || len(msg) < 2 || msg[0] != '{' || msg[len(msg)-1] != '}' {
+	if msgParser == nil || len(msg) < 2 || msg[0] != '{' {
 		fs.Add("_msg", msg)
 		return false
 	}
-	if msgParser != nil && len(msg) >= 2 && msg[0] == '{' && msg[len(msg)-1] == '}' {
-		if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msg), preserveKeys); err == nil {
-			fs.Fields = append(fs.Fields, msgParser.Fields...)
-			return true
+
+	// Trailing whitespace in the JSON message shouldn't break automatic JSON parsing.
+	//
+	// See https://github.com/VictoriaMetrics/VictoriaLogs/issues/1044
+	msgTrimmed := msg
+	if msg[len(msg)-1] != '}' {
+		msgTrimmed = trimTrailingJSONWhitespace(msg)
+		if len(msgTrimmed) < 2 || msgTrimmed[len(msgTrimmed)-1] != '}' {
+			fs.Add("_msg", msg)
+			return false
 		}
 	}
+
+	if err := msgParser.ParseLogMessage(bytesutil.ToUnsafeBytes(msgTrimmed), preserveKeys); err == nil {
+		fs.Fields = append(fs.Fields, msgParser.Fields...)
+		return true
+	}
 	fs.Add("_msg", msg)
 	return false
 }
 
+func trimTrailingJSONWhitespace(s string) string {
+	i := len(s)
+	for i > 0 {
+		switch s[i-1] {
+		case ' ', '\n', '\r', '\t':
+			i--
+		default:
+			return s[:i]
+		}
+	}
+	return ""
+}
+
 func parseLokiTimestamp(s string) (int64, error) {
 	if s == "" {
 		// Special case - an empty timestamp must be substituted with the current time by the caller.

app/vlinsert/loki/loki_json_test.go

@@ -198,4 +198,10 @@ func TestParseJSONRequest_ParseMessage(t *testing.T) {
 		}
 	]
 }`, []string{"a", "trace_id"}, []string{"x"}, []int64{1577836800000000001}, `{"x":"y","_msg":"111","parent_id":"abc","x":"{\"a\":123}"}`)
+
+	// with trailing whitespace in the JSON message
+	f(`{"streams":[{"stream":{"foo":"bar2"},"values":[["1577836800000000001","{\"bar\":\"baz\",\"_msg\":\"I am parsed\"}\n"]]}]}`,
+		nil, nil, []int64{1577836800000000001}, `{"foo":"bar2","bar":"baz","_msg":"I am parsed"}`)
+	f(`{"streams":[{"stream":{"foo":"bar2"},"values":[["1577836800000000001","{\"bar\":\"baz\",\"_msg\":\"I am parsed\"}\r\n\t  "]]}]}`,
+		nil, nil, []int64{1577836800000000001}, `{"foo":"bar2","bar":"baz","_msg":"I am parsed"}`)
 }

docs/victorialogs/CHANGELOG.md

@@ -30,6 +30,7 @@ according to the following docs:
 * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): add support for toggling line comments with `Ctrl/Cmd + /`. See [#1030](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1030).
 
 * BUGFIX: [`/select/logsql/hits` endpoint](https://docs.victoriametrics.com/victorialogs/querying/#querying-hits-stats): properly apply `offset` arg. Previously it resulted in incorrect calculations for the returned timestamps.
+* BUGFIX: [Loki data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/promtail/): properly parse JSON-encoded log fields inside Loki log message if it ends with whitespace chars such as `\n`. See [#1044](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1044).
 * BUGFIX: [dashboard/single-node](https://grafana.com/grafana/dashboards/22084): include `internalinsert` (`/internal/insert`) in `Logs ingestion rate` panels, so they work for data ingested via `vlagent` configured with `-remoteWrite.url=.../internal/insert`. See [#1053](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1053).
 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix autocomplete insertion to prevent overwriting remaining input and preserve newline characters. See [#917](https://github.com/VictoriaMetrics/VictoriaLogs/issues/917).
 * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): fix unexpected query expression reset to the previous value when clicking on "Hide chart" button. See [#1063](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1063).