vector: CPU increase and memory leak

A note for the community

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

Problem

Hello Vector team We see CPU increasement and a memory leak only in one of the agent pods. We are using the agent as DaemonSets --> a vector pod for each node. We are using the aggregator as StatefulSets. The agent is sending the k8s logs to the aggregator. Is that a bug? or did we configure it wrong?

Configuration

affinity: {}
  args:
  - -w
  - --config-dir
  - /etc/vector/
  - --log-format
  - json
  - -vv
  autoscaling:
    behavior: {}
    customMetric: {}
    enabled: false
    maxReplicas: 10
    minReplicas: 1
    targetCPUUtilizationPercentage: 80
  command: []
  commonLabels: {}
  containerPorts: []
  customConfig:
    api:
      enabled: false
    data_dir: /vector-data-dir
    sinks:
      prom_exporter:
        address: 0.0.0.0:9598
        inputs:
        - internal_metrics
        type: prometheus_exporter
      splunk_hec_dev_test:
        buffer:
          type: memory
        compression: gzip
        default_token: cf69e945-a1b2-a1b2-a1b2-be6777bea1b2
        encoding:
          codec: json
        endpoint: https://hec.a1b2.a234.log.cde.net.abc:443
        index: ugw_statistics
        inputs:
        - transform_remap_gateway_proxy_accesslog_test
        type: splunk_hec_logs
      vector_aggregator:
        address: abc-efgh-vector-aggregator.vector-aggregator.svc.cluster.local:7500
        inputs:
        - kubernetes_logs
        type: vector
    sources:
      internal_metrics:
        scrape_interval_secs: 2
        type: internal_metrics
      kubernetes_logs:
        glob_minimum_cooldown_ms: 1000
        max_read_bytes: 8192
        type: kubernetes_logs
    transforms:
      transform_remap_gateway_proxy_accesslog:
        inputs:
        - transform_route_gateway_proxy.access_log
        source: |
          ., err = parse_json(.message)
          if (err != null) {
            log("Remap gateway-proxy access_log, Unable to parse json: " + err, "error")
            abort
          }
          if (!exists(.targetLog) || .targetLog != "ABC-API-INSIGHTS-ACCESS-LOG") {
            abort
          }
          del(.targetLog)
          # verify insightsFields is not empty
          if (!exists(.insightsFields) || .insightsFields == null || .insightsFields == "") {
            log("Remap gateway-proxy access_log, insightsFields field is missing.")
            abort
          }
          result, err = merge(., .insightsFields)
          if (err != null) {
              log("Remap gateway-proxy access_log, Unable to merge insightsFields: " + err, "error")
          } else {
            . = result
          }
          del(.insightsFields)
          if (!exists(.insightsProfile) || .insightsProfile == null || .insightsProfile == "") {
            log("Remap gateway-proxy access_log, insightsProfile field is missing.")
            abort
          }
          path_parts, err = parse_regex(.path, r'(?P<pathname>[^?#]*)(?P<query>.*)')
          if (err != null) {
            log("Remap gateway-proxy access_log, Unable to parse regex: " + err, "error")
            abort
          }
          .path = path_parts.pathname
          if (.serviceTime != null) {
            .serviceTime = to_int!(.serviceTime)
          }
          if (.jwtZid != null) {
            .spcTenantId = .jwtZid
          }
          del(.jwtZid)
          if (.abcaaTenantId != null) {
            .spcTenantId = .abcaaTenantId
          }
          del(.abcaaTenantId)
          if (.jwtValStatus == null || .jwtSkipFailureReporting == true) {
            del(.jwtValStatus)
          }
          del(.jwtSkipFailureReporting)
          if (.rateLimitLimit == null) {
            del(.rateLimitLimit)
          }
          if (.rateLimitRemaining == null) {
            del(.rateLimitRemaining)
          }
          if (.rateLimitReset == null) {
            del(.rateLimitReset)
          }
          if (.rateLimitPolicy == null) {
            del(.rateLimitPolicy)
          }
        type: remap
      transform_remap_gateway_proxy_accesslog_all:
        inputs:
        - transform_remap_gateway_proxy_accesslog
        source: |
          if (.insightsProfile == "vector-e2e-test") {
            abort
          }
        type: remap
      transform_remap_gateway_proxy_accesslog_test:
        inputs:
        - transform_remap_gateway_proxy_accesslog
        source: |
          if (.insightsProfile != "vector-e2e-test") {
            abort
          }
          del(.insightsProfile)
        type: remap
      transform_route_gateway_proxy:
        inputs:
        - kubernetes_logs
        route:
          '*': .kubernetes.container_name == "gateway-proxy"
          access_log: starts_with(string!(.message), "{") && ends_with(string!(.message),
            "}")
        type: route
  dataDir: ""
  defaultVolumeMounts:
  - mountPath: /var/log/
    name: var-log
    readOnly: true
  - mountPath: /var/lib
    name: var-lib
    readOnly: true
  defaultVolumes:
  - hostPath:
      path: /var/log/
    name: var-log
  - hostPath:
      path: /var/lib/
    name: var-lib
  dnsConfig: {}
  dnsPolicy: ClusterFirst
  env: []
  envFrom: []
  existingConfigMaps: []
  extraContainers: []
  extraVolumeMounts: []
  extraVolumes:
  - name: secret-volume
    secret:
      secretName: vector-values
  fullnameOverride: ""
  global: {}
  haproxy:
    affinity: {}
    autoscaling:
      customMetric: {}
      enabled: false
      maxReplicas: 10
      minReplicas: 1
      targetCPUUtilizationPercentage: 80
    containerPorts: []
    customConfig: ""
    enabled: false
    existingConfigMap: ""
    extraContainers: []
    extraVolumeMounts: []
    extraVolumes: []
    image:
      pullPolicy: IfNotPresent
      pullSecrets: []
      repository: haproxytech/haproxy-alpine
      tag: 2.6.12
    initContainers: []
    livenessProbe:
      tcpSocket:
        port: 1024
    nodeSelector: {}
    podAnnotations: {}
    podLabels: {}
    podPriorityClassName: ""
    podSecurityContext: {}
    readinessProbe:
      tcpSocket:
        port: 1024
    replicas: 1
    resources: {}
    rollWorkload: true
    securityContext: {}
    service:
      annotations: {}
      externalTrafficPolicy: ""
      ipFamilies: []
      ipFamilyPolicy: ""
      loadBalancerIP: ""
      ports: []
      topologyKeys: []
      type: ClusterIP
    serviceAccount:
      annotations: {}
      automountToken: true
      create: true
    strategy: {}
    terminationGracePeriodSeconds: 60
    tolerations: []
  image:
    pullPolicy: IfNotPresent
    pullSecrets:
    - name: vector
    repository: build-releases-external.common.cdn.repositories.cloud.abc/timberio/vector
    tag: 0.32.1-distroless-libc
  ingress:
    annotations: {}
    className: ""
    enabled: false
    hosts: []
    tls: []
  initContainers: []
  lifecycle: {}
  livenessProbe: {}
  logLevel: info
  minReadySeconds: 0
  nameOverride: ""
  nodeSelector: {}
  persistence:
    accessModes:
    - ReadWriteOnce
    enabled: false
    existingClaim: ""
    finalizers:
    - kubernetes.io/pvc-protection
    hostPath:
      enabled: true
      path: /var/lib/vector
    selectors: {}
    size: 10Gi
  podAnnotations: {}
  podDisruptionBudget:
    enabled: false
    minAvailable: 1
  podHostNetwork: false
  podLabels:
    sidecar.istio.io/inject: "true"
    vector.dev/exclude: "true"
  podManagementPolicy: OrderedReady
  podMonitor:
    additionalLabels: {}
    enabled: false
    honorLabels: false
    honorTimestamps: true
    jobLabel: app.kubernetes.io/name
    metricRelabelings: []
    path: /metrics
    port: prom-exporter
    relabelings: []
  podPriorityClassName: ""
  podSecurityContext: {}
  psp:
    create: false
  rbac:
    create: true
  readinessProbe: {}
  replicas: 1
  resources:
    limits:
      cpu: 1000m
      memory: 2Gi
    requests:
      cpu: 10m
      memory: 128Mi
  role: Agent
  rollWorkload: false
  secrets:
    generic: {}
  securityContext: {}
  service:
    annotations: {}
    enabled: true
    externalTrafficPolicy: ""
    ipFamilies: []
    ipFamilyPolicy: ""
    loadBalancerIP: ""
    ports: []
    topologyKeys: []
    type: ClusterIP
  serviceAccount:
    annotations: {}
    automountToken: true
    create: true
  serviceHeadless:
    enabled: true
  terminationGracePeriodSeconds: 60
  tolerations:
  - effect: NoSchedule
    key: WorkGroup
    operator: Equal
    value: abproxy
  - effect: NoExecute
    key: WorkGroup
    operator: Equal
    value: abproxy
  topologySpreadConstraints: []
  ugwSecretConfigEnabled: false
  updateStrategy: {}
  workloadResourceAnnotations: {}

Version

0.32.1-distroless-libc

Debug Output

No response

Example Data

No response

Additional Context

No response

References

No response