VictoriaLogs: VictoriaLogs Cluster: missing _msg field for any ingest API

Is your question request related to a specific component?

vector config

[sources.kubernetes_logs]

    [transforms.parse_logs]
    type = "remap"
    inputs = ["kubernetes_logs"]
    source = """
    json_parsed, err = parse_json(.message)
    if err == null && json_parsed != null {
      .= merge!(., json_parsed)
    } else {
      nginx_log, err1 = parse_nginx_log(.message, "combined")
      if err1 == null && nginx_log != null {
        .= merge(., nginx_log)
      }
    }

    .app = .kubernetes.pod_labels.app
    .container_name = .kubernetes.container_name
    .pod_name = .kubernetes.pod_name
    .namespace = .kubernetes.pod_namespace

    log_level, err = get(., ["level"])
    if err == null && log_level != null {
      . = set!(., ["level"], log_level)
    } else {
      . = set!(., ["level"], "info")
    }

    del(.kubernetes)
    del(.file)
    del(.cost)
    del(.stream)
    del(.source_type)
    """
    [sinks.vmlog]
    type = "loki"
    inputs = ["parse_logs"]
    endpoint = "http://vlinsert.ops.svc:9481/insert/loki/"
    path = "/api/v1/push?_stream_fields=app,pod_name,container_name,namespace,level&_msg_field=message&_time_field=timestamp&disable_message_parsing=1"
    compression = "none"
    out_of_order_action = "accept"
    remove_label_fields = true
    tenant_id = "1"
    [sinks.vmlog.request]
    concurrency = 10
    rate_limit_num = 500
    retry_attempts = 3
    [sinks.vmlog.healthcheck]
    enabled = false
    [sinks.vmlog.labels]
    "app" = "{{ .app }}"
    "container_name" = "{{ .container_name }}"
    "pod_name" = "{{ .pod_name }}"
    "namespace" = "{{ .namespace }}"
    "level" = "{{ .level }}"
    [sinks.vmlog.encoding]
    codec = "raw_message"

vlstorage

Image vminsert

Image

Describe the question in detail

I migrated from loki to VictoriaLogs, why does it prompt me that I don’t have the msg field?

Image Image

Troubleshooting docs

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Comments: 27 (14 by maintainers)

Commits related to this issue

Most upvoted comments

Hi,

I have the same problem with the syslog input on version “v1.18.0-victorialogs” In a single node variant everything works as expected. In a cluster consisting of vlinsert/vlselect + vlstorage i get the “missing _msg field” error.

Here’s a log from the vlstorage node:

root@s3 ~# su -s /bin/bash -c '/usr/local/bin/victoria-logs-prod -envflag.enable -httpListenAddr=:9491 -storageDataPath=/var/lib/victoria-logs/ -retention.maxDiskSpaceUsag
eBytes=1GB -storage.minFreeDiskSpaceBytes=1GB -logIngestedRows' victoriametrics
2025-04-14T18:14:30.655Z        info    VictoriaMetrics/lib/logger/flag.go:12   build version: victoria-logs-20250410-150037-tags-v1.18.0-victorialogs-0-g7a46af3920
2025-04-14T18:14:30.655Z        info    VictoriaMetrics/lib/logger/flag.go:13   command-line flags
2025-04-14T18:14:30.656Z        info    VictoriaMetrics/lib/logger/flag.go:20     -envflag.enable="true"
2025-04-14T18:14:30.656Z        info    VictoriaMetrics/lib/logger/flag.go:20     -httpListenAddr=":9491"
2025-04-14T18:14:30.656Z        info    VictoriaMetrics/lib/logger/flag.go:20     -logIngestedRows="true"
2025-04-14T18:14:30.656Z        info    VictoriaMetrics/lib/logger/flag.go:20     -retention.maxDiskSpaceUsageBytes="1GB"
2025-04-14T18:14:30.656Z        info    VictoriaMetrics/lib/logger/flag.go:20     -storage.minFreeDiskSpaceBytes="1GB"
2025-04-14T18:14:30.657Z        info    VictoriaMetrics/lib/logger/flag.go:20     -storageDataPath="/var/lib/victoria-logs/"
2025-04-14T18:14:30.657Z        info    VictoriaMetrics/app/victoria-logs/main.go:42    starting VictoriaLogs at "[:9491]"...
2025-04-14T18:14:30.657Z        info    VictoriaMetrics/app/vlstorage/main.go:108       opening storage at -storageDataPath=/var/lib/victoria-logs/
2025-04-14T18:14:30.660Z        info    VictoriaMetrics/lib/memory/memory.go:42 limiting caches to 610295808 bytes, leaving 406863872 bytes to the OS according to -memory.allowedPercent=60
2025-04-14T18:14:30.682Z        info    VictoriaMetrics/app/vlstorage/main.go:114       successfully opened storage in 0.025 seconds; smallParts: 5; bigParts: 0; smallPartBlocks: 5; bigPartBlocks: 0; smallPartRows: 53; bigPartRows: 0; smallPartSize: 2902 bytes; bigPartSize: 0 bytes
2025-04-14T18:14:30.683Z        info    VictoriaMetrics/app/victoria-logs/main.go:50    started VictoriaLogs in 0.026 seconds; see https://docs.victoriametrics.com/victorialogs/
2025-04-14T18:14:30.683Z        info    VictoriaMetrics/lib/httpserver/httpserver.go:170        started server at http://0.0.0.0:9491/
2025-04-14T18:14:30.684Z        info    VictoriaMetrics/lib/httpserver/httpserver.go:172        pprof handlers are exposed at http://0.0.0.0:9491/debug/pprof/
2025-04-14T18:14:43.614Z        info    VictoriaMetrics/lib/logstorage/partition.go:157 partition /var/lib/victoria-logs/partitions/20250414: new log entry {"_msg":"missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field","_msg":"error: asdf","_stream":"{app_name=\"development\",hostname=\"MacBook-Pro\"}","_time":"2025-04-14T18:14:43.21708764Z","app_name":"development","facility":"16","format":"rfc5424","hostname":"MacBook-Pro","msg_id":"-","priority":"131","proc_id":"55180","severity":"3","timestamp":"2025-04-14T18:14:43.141+03:00"}
2025-04-14T18:14:43.614Z        info    VictoriaMetrics/lib/logstorage/partition.go:157 partition /var/lib/victoria-logs/partitions/20250414: new log entry {"_msg":"missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field","_msg":"error: asdf","_stream":"{app_name=\"development\",hostname=\"MacBook-Pro\"}","_time":"2025-04-14T18:14:43.263416816Z","app_name":"development","facility":"16","format":"rfc5424","hostname":"MacBook-Pro","msg_id":"-","priority":"131","proc_id":"55180","severity":"3","timestamp":"2025-04-14T18:14:43.189+03:00"}
2025-04-14T18:14:44.614Z        info    VictoriaMetrics/lib/logstorage/partition.go:157 partition /var/lib/victoria-logs/partitions/20250414: new log entry {"_msg":"missing _msg field; see https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field","_msg":"error: asdf","_stream":"{app_name=\"development\",hostname=\"MacBook-Pro\"}","_time":"2025-04-14T18:14:43.745900973Z","app_name":"development","facility":"16","format":"rfc5424","hostname":"MacBook-Pro","msg_id":"-","priority":"131","proc_id":"55180","severity":"3","timestamp":"2025-04-14T18:14:43.671+03:00"}

The vlinsert/vlselect node is started using the following command, but doesn’t output any logs.

root@s3 ~# su -s /bin/bash -c '/usr/local/bin/victoria-logs-prod -envflag.enable -httpListenAddr=:9494 -storageNode=s3:9491 -syslog.listenAddr.udp=:1514 -syslog.tenantID.udp=0:0 -syslog.streamFields.udp=\'["hostname","app_name"]\' -syslog.useLocalTimestamp.udp=true -logIngestedRows' victoriametrics
2025-04-14T18:33:02.923Z        info    VictoriaMetrics/lib/logger/flag.go:12   build version: victoria-logs-20250410-150037-tags-v1.18.0-victorialogs-0-g7a46af3920
2025-04-14T18:33:02.923Z        info    VictoriaMetrics/lib/logger/flag.go:13   command-line flags
2025-04-14T18:33:02.924Z        info    VictoriaMetrics/lib/logger/flag.go:20     -envflag.enable="true"
2025-04-14T18:33:02.924Z        info    VictoriaMetrics/lib/logger/flag.go:20     -httpListenAddr=":9494"
2025-04-14T18:33:02.924Z        info    VictoriaMetrics/lib/logger/flag.go:20     -logIngestedRows="true"
2025-04-14T18:33:02.924Z        info    VictoriaMetrics/lib/logger/flag.go:20     -storageNode="s3:9491"
2025-04-14T18:33:02.925Z        info    VictoriaMetrics/lib/logger/flag.go:20     -syslog.listenAddr.udp=":1514"
2025-04-14T18:33:02.925Z        info    VictoriaMetrics/lib/logger/flag.go:20     -syslog.streamFields.udp="\"[\\\"hostname\\\",\\\"app_name\\\"]\""
2025-04-14T18:33:02.925Z        info    VictoriaMetrics/lib/logger/flag.go:20     -syslog.tenantID.udp="0:0"
2025-04-14T18:33:02.925Z        info    VictoriaMetrics/lib/logger/flag.go:20     -syslog.useLocalTimestamp.udp="true"
2025-04-14T18:33:02.926Z        info    VictoriaMetrics/app/victoria-logs/main.go:42    starting VictoriaLogs at "[:9494]"...
2025-04-14T18:33:02.926Z        info    VictoriaMetrics/app/vlstorage/main.go:139       starting insert service for nodes [s3:9491]
2025-04-14T18:33:02.926Z        info    VictoriaMetrics/app/vlstorage/main.go:142       initializing select service for nodes [s3:9491]
2025-04-14T18:33:02.926Z        info    VictoriaMetrics/app/vlstorage/main.go:145       initialized all the network services
2025-04-14T18:33:02.926Z        info    VictoriaMetrics/app/victoria-logs/main.go:50    started VictoriaLogs in 0.000 seconds; see https://docs.victoriametrics.com/victorialogs/
2025-04-14T18:33:02.927Z        info    VictoriaMetrics/app/vlinsert/syslog/syslog.go:203       started accepting syslog messages at -syslog.listenAddr.udp=":1514"
2025-04-14T18:33:02.927Z        info    VictoriaMetrics/lib/httpserver/httpserver.go:170        started server at http://0.0.0.0:9494/
2025-04-14T18:33:02.927Z        info    VictoriaMetrics/lib/httpserver/httpserver.go:172        pprof handlers are exposed at http://0.0.0.0:9494/debug/pprof/

Thanks! If I’m not mistaken, the following URL

/api/v1/push?_stream_fields=app,pod_name,container_name,namespace,level&_msg_field=message&_time_field=timestamp&disable_message_parsing=1

expects to see the following fields in your log: [app,pod_name,container_name,namespace,level, message, timestamp]. And in the logs you provided, it seems that they don’t have these fields. Is there something I’m missing?

Hi. This is my configuration. I use vector to collect the logs of a service and write them to both v1.15.0 and v1.18.0 clusters. I don’t know why the cluster version reports missing the _msg field.

    [sinks.vmlog]
    type = "loki"
    inputs = ["parse_logs"]
    endpoint = "http://vlinsert.ops.svc:9481/insert/loki/"
    path = "/api/v1/push?_stream_fields=app,pod_name,container_name,namespace,level&_msg_field=message&_time_field=timestamp&disable_message_parsing=1"
    compression = "none"
    out_of_order_action = "accept"
    remove_label_fields = true
    tenant_id = "1"
    [sinks.vmlog.request]
    concurrency = 10
    rate_limit_num = 500
    retry_attempts = 3
    [sinks.vmlog.healthcheck]
    enabled = false
    [sinks.vmlog.labels]
    "app" = "{{ .app }}"
    "container_name" = "{{ .container_name }}"
    "pod_name" = "{{ .pod_name }}"
    "namespace" = "{{ .namespace }}"
    "level" = "{{ .level }}"
    [sinks.vmlog.encoding]
    codec = "raw_message"

    [sinks.vmlogsingle]
    type = "loki"
    inputs = ["parse_logs"]
    endpoint = "http://vmlog.ops.svc:9428/insert/loki/"
    path = "/api/v1/push?_stream_fields=app,pod_name,container_name,namespace,level&_msg_field=message&_time_field=timestamp&disable_message_parsing=1"
    compression = "none"
    out_of_order_action = "accept"
    remove_label_fields = true
    tenant_id = "1"
    [sinks.vmlogsingle.request]
    concurrency = 10
    rate_limit_num = 500
    retry_attempts = 3
    [sinks.vmlogsingle.healthcheck]
    enabled = false
    [sinks.vmlogsingle.labels]
    "app" = "{{ .app }}"
    "container_name" = "{{ .container_name }}"
    "pod_name" = "{{ .pod_name }}"
    "namespace" = "{{ .namespace }}"
    "level" = "{{ .level }}"
    [sinks.vmlogsingle.encoding]
    codec = "raw_message"

Image Image

Hi. It’s important to know what data was sent to VictoriaLogs and how the request looks like. Could you provide a request example to VictoriaLogs /insert/loki/api/v1/push API, or print the data via vector console sink?

Hi. Here is some vector console sink data

vector_logs.txt

Image