frigate: [Support]: 2d52fbd-tensorrt detection KeyError: -16

Describe the problem you are having

After moving from 12d4a47-tensorrt to 2d52fbd-tensorrt detection is no longer working. All cameras show “no frames have been received”. Clicking on one of the cameras does show the live preview / Go2RTC works. Loading birdseye generates the following error:

frigate    | 2023-07-13 11:42:57.443603455  11:42:57.443 ERR [exec] timeout url="exec:ffmpeg -hide_banner -f rawvideo -pix_fmt yuv420p -video_size 1280x720 -r 10 -i /tmp/cache/birdseye -c:v h264_nvenc -g 50 -profile:v high -level:v auto -preset:v p2 -tune:v ll -rtsp_transport tcp -f rtsp {output}"

Cameras throw errors like this:

2023-07-13 11:17:12.914464908  Process camera_processor:front_door:
2023-07-13 11:17:12.916049239  Traceback (most recent call last):
2023-07-13 11:17:12.916371989    File "/usr/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
2023-07-13 11:17:12.916374419      self.run()
2023-07-13 11:17:12.916375759    File "/usr/lib/python3.9/multiprocessing/process.py", line 108, in run
2023-07-13 11:17:12.916376849      self._target(*self._args, **self._kwargs)
2023-07-13 11:17:12.916378129    File "/opt/frigate/frigate/video.py", line 507, in track_camera
2023-07-13 11:17:12.916379069      process_frames(
2023-07-13 11:17:12.916380179    File "/opt/frigate/frigate/video.py", line 882, in process_frames
2023-07-13 11:17:12.916381049      detect(
2023-07-13 11:17:12.916388279    File "/opt/frigate/frigate/video.py", line 581, in detect
2023-07-13 11:17:12.916389449      region_detections = object_detector.detect(tensor_input)
2023-07-13 11:17:12.916401219    File "/opt/frigate/frigate/object_detection.py", line 223, in detect
2023-07-13 11:17:12.916402419      (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
2023-07-13 11:17:12.916403279  KeyError: -16

Version

0.13.0-2d52fbd

Frigate config file

telemetry:
  version_check: false

birdseye:
  enabled: true
  restream: true
  mode: continuous

audio:
  listen:
    - bark
    - scream
    - speech
    - yell

logger:
  default: info
  #default: debug
  #logs:
  #  frigate.ffmpeg: debug
  #  frigate.go2rtc: debug
  #  frigate.record: debug

database:
  path: /db/frigate.db

mqtt:
  host: mqtt
  port: 8883
  tls_ca_certs: /config/CA.pem
  user: frigate
  password: pass


ffmpeg:
  hwaccel_args: preset-nvidia-h264
  output_args:
    record: preset-record-generic-audio-aac

detectors:
  #cpu:
  #  type: cpu
  #  num_threads: 4
  tensorrt:
    type: tensorrt
    device: 0

model:
  path: /config/model_cache/tensorrt/yolov7-tiny-416.trt
  input_tensor: nchw
  input_pixel_format: rgb
  width: 416
  height: 416

detect:
  fps: 5
  enabled: True
  max_disappeared: 25

snapshots:
  enabled: true
  quality: 100
  retain:
    default: 30

motion:
  threshold: 25
  contour_area: 30
  delta_alpha: 0.2
  frame_alpha: 0.2
  frame_height: 50

record:
  enabled: true
  expire_interval: 5
  retain:
    days: 3
    mode: all
  events:
    pre_capture: 30
    post_capture: 30
    retain:
      default: 30
      mode: all

go2rtc:
  #log:
  #  exec: trace
  rtsp:
    username: "admin"
    password: "pass"
  webrtc:
    candidates:
      - x.x.x.20:8555
  streams:
    front_door:
      - "rtsp://user:pass@x.x.x.68:554/Streaming/Channels/101"
      - "ffmpeg:front_door#audio=opus#audio=aac"
    front_door_detect:
      - "rtsp://user:pass@x.x.x.68:554/Streaming/Channels/103"
      - "ffmpeg:front_door_detect#audio=opus#audio=aac"
    garage_door:
      - "rtsp://user:pass@x.x.x.69:554/Streaming/Channels/101"
      - "ffmpeg:garage_door#audio=opus#audio=aac"
    garage_door_detect:
      - "rtsp://user:pass@x.x.x.69:554/Streaming/Channels/103"
      - "ffmpeg:garage_door_detect#audio=opus#audio=aac"
    other_camera:
      - "rtsp://user:pass@x.x.x.79:554/Streaming/Channels/101"
    other_camera_detect:
      - "rtsp://user:pass@x.x.x.79:554/Streaming/Channels/103"
    garage_interior:
      - 'rtsp://user:pass@x.x.x.66/1'
    garage_interior_detect:
      - 'rtsp://user:pass@x.x.x.66/4'
    foscam:
      - "rtsp://user:pass@x.x.x.65:88/videoMain"
      - "ffmpeg:foscam#audio=aac#hardware"
    foscam_detect:
      - "rtsp://user:pass@x.x.x.65:88/videoSub"
      - "ffmpeg:foscam_detect#audio=aac#hardware"

cameras:
  front_door:
    enabled: true
    audio:
      enabled: false
    detect:
      width: 720
      height: 1280
      enabled: true
    birdseye:
      enabled: True
      order: 0
    mqtt:
      enabled: true
      crop: True
      quality: 100
      height: 500
      timestamp: False
      bounding_box: False
    zones:
      zone_0:
        coordinates: 0,204,0,32,359,31,356,0,720,0,720,1280,0,1280
    live:
       stream_name: front_door_detect
    ffmpeg:
      inputs:
        - path: "rtsp://127.0.0.1:8554/front_door"
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: "rtsp://127.0.0.1:8554/front_door_detect"
          input_args: preset-rtsp-restream
          roles:
            - detect
            - audio
    objects:
      filters:
        person:
          mask:
            - 90,0,720,0,720,112,720,98,599,57,125,184
      track:
        - person
        - car
        - dog
        - cat
        - motorcycle
        - bus
        - bicycle
        - backpack
        - handbag
        - suitcase
        - bird
        - bear
        - truck

  foscam:
    enabled: true
    audio:
      enabled: false
    detect:
      width: 1280
      height: 720
      enabled: true
    birdseye:
      enabled: true
      order: 2
    onvif:
      host: x.x.x.65
      port: 888
      user: user
      password: "pass"
    mqtt:
      enabled: true
      crop: True
      quality: 100
      height: 500
      timestamp: False
      bounding_box: False
    live:
       stream_name: foscam_detect
    ffmpeg:
      inputs:
        - path: rtsp://127.0.0.1:8554/foscam
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/foscam_detect
          input_args: preset-rtsp-restream
          roles:
            - detect
            - audio
    objects:
      track:
        - cat
        #- person
        #- car
        #- cat
        #- motorcycle
        #- bus
        
  garage_door:
    enabled: true
    audio:
      enabled: false
    detect:
      width: 1280
      height: 720
      enabled: true
    birdseye:
      enabled: True
      order: 1
    mqtt:
      enabled: true
      crop: True
      quality: 100
      height: 500
      timestamp: False
      bounding_box: False
    zones:
      zone_0:
        coordinates: 371,0,1280,0,1280,720,593,720,0,720,0,34,369,38
    live:
       stream_name: garage_door_detect
    ffmpeg:
      inputs:
        - path: "rtsp://127.0.0.1:8554/garage_door"
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: "rtsp://127.0.0.1:8554/garage_door_detect"
          input_args: preset-rtsp-restream
          roles:
            - detect
            - audio
    objects:
      filters:
        car:
          mask:
            - 654,177,1280,239,1280,720,0,720,0,0,1280,0,1280,227,1078,234,1062,64,760,88
        person:
          mask:
            - 0,350,0,0,1280,0,1280,166,1215,61,1069,58,624,65
      track:
        - person
        - car
        - dog
        - cat
        - motorcycle
        - bus
        - bicycle
        - backpack
        - handbag
        - suitcase
        - bird
        - bear
        - truck

  garage_interior:
    enabled: true
    audio:
      enabled: false
    detect:
      width: 1280
      height: 720
      enabled: true
    birdseye:
      enabled: False
    mqtt:
      enabled: true
      crop: True
      quality: 100
      height: 500
      timestamp: False
      bounding_box: False
    zones:
      zone_0:
        coordinates: 1280,0,210,0,212,35,0,36,0,720,1280,720
    live:
       stream_name: garage_interior_detect
    ffmpeg:
      inputs:
        - path: rtsp://127.0.0.1:8554/garage_interior
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/garage_interior_detect
          input_args: preset-rtsp-restream
          roles:
            - detect
    objects:
      filters:
        car:
          mask:
            - 1280,0,1280,720,0,720,0,415,569,128,564,0
      track:
        - person
        - dog
        - cat
        - bus
        - backpack
        - handbag
        - suitcase
        - bird
        - bear

  other_camera:
    enabled: false
    audio:
      enabled: false
    detect:
      width: 1280
      height: 720
      enabled: true
    birdseye:
      enabled: true
    mqtt:
      enabled: true
      crop: True
      quality: 100
      height: 500
      timestamp: False
      bounding_box: False
    live:
       stream_name: other_camera_detect
    ffmpeg:
      inputs:
        - path: rtsp://127.0.0.1:8554/other_camera
          input_args: preset-rtsp-restream
          roles:
            - record
        - path: rtsp://127.0.0.1:8554/other_camera_detect
          input_args: preset-rtsp-restream
          roles:
            - detect
            - audio
    objects:
      track:
        - person
        - cat
        - backpack
        - handbag

Relevant log output

2023-07-13 11:17:05.257367471  [INFO] Preparing Frigate...
2023-07-13 11:17:05.276831744  [INFO] Starting Frigate...
2023-07-13 11:17:06.882436359  [2023-07-13 11:17:06] frigate.app                    INFO    : Starting Frigate (0.13.0-2d52fbd)
2023-07-13 11:17:07.065467494  [2023-07-13 11:17:07] peewee_migrate.logs            INFO    : Starting migrations
2023-07-13 11:17:07.071427747  [2023-07-13 11:17:07] peewee_migrate.logs            INFO    : There is nothing to migrate
2023-07-13 11:17:07.268091100  [2023-07-13 11:17:07] frigate.app                    INFO    : Recording process started: 317
2023-07-13 11:17:07.270568892  [2023-07-13 11:17:07] frigate.app                    INFO    : go2rtc process pid: 98
2023-07-13 11:17:07.300852871  [2023-07-13 11:17:07] detector.tensorrt              INFO    : Starting detection process: 326
2023-07-13 11:17:07.306154624  [2023-07-13 11:17:07] frigate.app                    INFO    : Output process started: 327
2023-07-13 11:17:07.318445822  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor started for front_door: 333
2023-07-13 11:17:07.327946308  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor started for foscam: 335
2023-07-13 11:17:07.360832478  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor started for garage_door: 336
2023-07-13 11:17:07.367168402  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor started for garage_interior: 338
2023-07-13 11:17:07.367178952  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor not started for disabled camera other_camera
2023-07-13 11:17:07.367182042  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor not started for disabled camera livingroom
2023-07-13 11:17:07.367184972  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor not started for disabled camera jeff_laptop
2023-07-13 11:17:07.367203012  [2023-07-13 11:17:07] frigate.app                    INFO    : Camera processor not started for disabled camera jeff_x1carbon
2023-07-13 11:17:07.375992537  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process started for front_door: 339
2023-07-13 11:17:07.393823869  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process started for foscam: 343
2023-07-13 11:17:07.402425934  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process started for garage_door: 348
2023-07-13 11:17:07.416600723  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process started for garage_interior: 351
2023-07-13 11:17:07.416850933  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process not started for disabled camera other_camera
2023-07-13 11:17:07.416853483  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process not started for disabled camera livingroom
2023-07-13 11:17:07.416855053  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process not started for disabled camera jeff_laptop
2023-07-13 11:17:07.416867473  [2023-07-13 11:17:07] frigate.app                    INFO    : Capture process not started for disabled camera jeff_x1carbon
2023-07-13 11:17:07.422958207  [2023-07-13 11:17:07] frigate.app                    INFO    : Audio process started: 354
2023-07-13 11:17:07.840209077  [2023-07-13 11:17:07] frigate.detectors.plugins.tensorrt INFO    : Loaded engine size: 35 MiB
2023-07-13 11:17:08.376530718  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +193, GPU +74, now: CPU 549, GPU 598 (MiB)
2023-07-13 11:17:08.553135606  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuDNN: CPU +111, GPU +46, now: CPU 660, GPU 644 (MiB)
2023-07-13 11:17:08.555256537  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +34, now: CPU 0, GPU 34 (MiB)
2023-07-13 11:17:08.558485679  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 625, GPU 636 (MiB)
2023-07-13 11:17:08.560319900  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 625, GPU 644 (MiB)
2023-07-13 11:17:08.561097251  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt INFO    : [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +13, now: CPU 0, GPU 47 (MiB)
2023-07-13 11:17:08.561274631  [2023-07-13 11:17:08] frigate.detectors.plugins.tensorrt WARNING : CUDA lazy loading is not enabled. Enabling it can significantly reduce device memory usage. See `CUDA_MODULE_LOADING` in https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars
2023-07-13 11:17:12.914464908  Process camera_processor:front_door:
2023-07-13 11:17:12.916049239  Traceback (most recent call last):
2023-07-13 11:17:12.916371989    File "/usr/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
2023-07-13 11:17:12.916374419      self.run()
2023-07-13 11:17:12.916375759    File "/usr/lib/python3.9/multiprocessing/process.py", line 108, in run
2023-07-13 11:17:12.916376849      self._target(*self._args, **self._kwargs)
2023-07-13 11:17:12.916378129    File "/opt/frigate/frigate/video.py", line 507, in track_camera
2023-07-13 11:17:12.916379069      process_frames(
2023-07-13 11:17:12.916380179    File "/opt/frigate/frigate/video.py", line 882, in process_frames
2023-07-13 11:17:12.916381049      detect(
2023-07-13 11:17:12.916388279    File "/opt/frigate/frigate/video.py", line 581, in detect
2023-07-13 11:17:12.916389449      region_detections = object_detector.detect(tensor_input)
2023-07-13 11:17:12.916401219    File "/opt/frigate/frigate/object_detection.py", line 223, in detect
2023-07-13 11:17:12.916402419      (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
2023-07-13 11:17:12.916403279  KeyError: -16
2023-07-13 11:17:13.500711516  Process camera_processor:garage_interior:
2023-07-13 11:17:13.501819937  Traceback (most recent call last):
2023-07-13 11:17:13.501840667    File "/usr/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
2023-07-13 11:17:13.501842167      self.run()
2023-07-13 11:17:13.501843457    File "/usr/lib/python3.9/multiprocessing/process.py", line 108, in run
2023-07-13 11:17:13.501844597      self._target(*self._args, **self._kwargs)
2023-07-13 11:17:13.501845707    File "/opt/frigate/frigate/video.py", line 507, in track_camera
2023-07-13 11:17:13.501846697      process_frames(
2023-07-13 11:17:13.501852187    File "/opt/frigate/frigate/video.py", line 882, in process_frames
2023-07-13 11:17:13.501853107      detect(
2023-07-13 11:17:13.501854127    File "/opt/frigate/frigate/video.py", line 581, in detect
2023-07-13 11:17:13.501855167      region_detections = object_detector.detect(tensor_input)
2023-07-13 11:17:13.501856297    File "/opt/frigate/frigate/object_detection.py", line 223, in detect
2023-07-13 11:17:13.501857357      (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
2023-07-13 11:17:13.501882597  KeyError: -13
2023-07-13 11:17:15.211183164  Process camera_processor:foscam:
2023-07-13 11:17:15.212125344  Traceback (most recent call last):
2023-07-13 11:17:15.212328634    File "/usr/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
2023-07-13 11:17:15.212330854      self.run()
2023-07-13 11:17:15.212332294    File "/usr/lib/python3.9/multiprocessing/process.py", line 108, in run
2023-07-13 11:17:15.212333444      self._target(*self._args, **self._kwargs)
2023-07-13 11:17:15.212334624    File "/opt/frigate/frigate/video.py", line 507, in track_camera
2023-07-13 11:17:15.212335604      process_frames(
2023-07-13 11:17:15.212336734    File "/opt/frigate/frigate/video.py", line 882, in process_frames
2023-07-13 11:17:15.212337624      detect(
2023-07-13 11:17:15.212338784    File "/opt/frigate/frigate/video.py", line 581, in detect
2023-07-13 11:17:15.212339904      region_detections = object_detector.detect(tensor_input)
2023-07-13 11:17:15.212354704    File "/opt/frigate/frigate/object_detection.py", line 223, in detect
2023-07-13 11:17:15.212355924      (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
2023-07-13 11:17:15.212356804  KeyError: -16
2023-07-13 11:17:17.016229347  Process camera_processor:garage_door:
2023-07-13 11:17:17.017311507  Traceback (most recent call last):
2023-07-13 11:17:17.017314887    File "/usr/lib/python3.9/multiprocessing/process.py", line 315, in _bootstrap
2023-07-13 11:17:17.017316247      self.run()
2023-07-13 11:17:17.017317457    File "/usr/lib/python3.9/multiprocessing/process.py", line 108, in run
2023-07-13 11:17:17.017318547      self._target(*self._args, **self._kwargs)
2023-07-13 11:17:17.017319577    File "/opt/frigate/frigate/video.py", line 507, in track_camera
2023-07-13 11:17:17.017320557      process_frames(
2023-07-13 11:17:17.017321657    File "/opt/frigate/frigate/video.py", line 882, in process_frames
2023-07-13 11:17:17.017322567      detect(
2023-07-13 11:17:17.017323587    File "/opt/frigate/frigate/video.py", line 581, in detect
2023-07-13 11:17:17.017324587      region_detections = object_detector.detect(tensor_input)
2023-07-13 11:17:17.017331977    File "/opt/frigate/frigate/object_detection.py", line 223, in detect
2023-07-13 11:17:17.017333027      (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
2023-07-13 11:17:17.017377227  KeyError: -20

FFprobe output from your camera

Foscam:
[{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"155/6","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","display_aspect_ratio":"16:9","height":1080,"width":1920},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}},{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"31/6","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","display_aspect_ratio":"16:9","height":720,"width":1280},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}}]

Front Door:
[{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"30/1","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","display_aspect_ratio":"95:168","height":2560,"width":1440},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}},{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"6/1","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","display_aspect_ratio":"95:168","height":1280,"width":720},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}}]

Garage Door:
[{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"30/1","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","display_aspect_ratio":"168:95","height":1440,"width":2560},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}},{"return_code":0,"stderr":"","stdout":{"programs":[],"streams":[{"avg_frame_rate":"6/1","codec_long_name":"H.264/AVC/MPEG-4AVC/MPEG-4part10","display_aspect_ratio":"168:95","height":720,"width":1280},{"avg_frame_rate":"0/0","codec_long_name":"AAC(AdvancedAudioCoding)"}]}}]

Frigate stats

No response

Operating system

Ubuntu 22.04

Install method

Docker Compose

Coral version

Other

Network connection

Wired

Camera make and model

Mixed

Any other information that may be helpful

docker-compose.yaml

services:
  frigate:
    container_name: frigate
    environment:
      - USE_FP16=false
    privileged: true
    restart: unless-stopped
    #image: ghcr.io/blakeblackshear/frigate:dev-12d4a47-tensorrt
    image: ghcr.io/blakeblackshear/frigate:dev-2d52fbd-tensorrt
    shm_size: "4096mb" # update for your cameras based on calculation above
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /home/frigate/config:/config
      - /home/frigate/db:/db
      - /mnt/recordings/frigate:/media/frigate
      - /mnt/recordings/temp:/tmp
    ports:
      - "127.0.0.1:5000:5000"
      - "x.x.x.20:8554:8554" # RTSP
      - "x.x.x.20:8555:8555" # WebRTC
    runtime: nvidia
    deploy:
        resources:
          reservations:
            devices:
              - driver: nvidia
                count: 1
                capabilities: [gpu]