pypylon: Another memory leak when using BaslerCompressionBeyond (this time when sending grab array to different thread)
Describe the issue:
When using BaslerCompressionBeyond and doing decompression within grabbing loop
while camera.IsGrabbing():
# Wait for an image and then retrieve it. A timeout of 5s is used.
with camera.RetrieveResult(5 * 1000, pylon.TimeoutHandling_Return) as grab:
# Image grabbed successfully?
if grab.GrabSucceeded():
try:
# Add frame to redis
f(decompressor.DecompressImage(grab).Array if decompressor else grab.Array)
tic += 1
except pylon.GenericException as e:
logger.debug(f"{key}> {e}")
else:
logger.debug(f"{key}> {grab.GetErrorDescription()}")
continue
and sending frame array to different thread (using ThreadPoolExecutor) – so grabbing loop is not blocked (in my case on storring image to redis) Memory Leak is happening – aka memory is increasing after each iteration through grabbing loop.
Not sure why it is happening since I am sending allready grabbed array and not buffer or whole grab object. Please see complete code attached.
Tried to use copy() as well to create new instance of grab array. but same scenario happens
When I disable threadpool i can no longer see Memory Leak.
Also — this does not happen in case BaslerCompressionBeyond is not used. Aka posting frame array to different thread. So it must be related to pylon.ImageDecompressor() or compression descriptor somehow?
UPDATE: this happens only when redis[hiredis] is used pip install redis[hiredis]
Reproduce the code example:
from argparse import ArgumentParser
from concurrent.futures.thread import ThreadPoolExecutor
from dataclasses import dataclass, field
from pickle import dumps
from time import time
from loguru import logger
from redis import ConnectionPool, Redis
from redis.exceptions import RedisError
from pypylon import pylon
def get_args():
parser = ArgumentParser("Basler camera driver")
parser.add_argument("--redis_host", type=str, default="redis.db", help="Hostname for redis")
parser.add_argument("--redis_key", type=str, default="basler", help="Key for redis")
parser.add_argument("--model", type=str, default=False, help="Camera model")
parser.add_argument("--address", type=str, default=False, help="Camera IP address")
parser.add_argument("--pfs", type=str, default="/app/pfs/basler.pfs", help="Camera nodemap file")
parser.add_argument("--fps", type=int, default=30, help="Basler FPS")
parser.add_argument("--rotate", type=lambda x: x.lower() in ["true", "1"], default=False, help="Rotate camera")
return parser.parse_args()
FLAGS = get_args()
TIC = 60
@dataclass(order=True)
class Camera:
# device
device: pylon.DeviceInfo
config: str
# info
model: str = field(init=False)
sn: int = field(init=False)
sort_index: int = field(init=False)
# camera
camera: pylon.InstantCamera = field(init=False)
format: str = field(init=False)
decompressor: pylon.ImageDecompressor | None = None
def __repr__(self):
return f"model: {self.model} type: {self.device.GetDeviceClass()} sn: {self.sn}"
# discover camera
def _pre_init(self):
# get camera info
self.model = self.device.GetModelName()
self.sn = self.device.GetSerialNumber()
# set sort index
self.sort_index = self.sn
logger.info(f"Added camera {self}")
# initialize camera
def _init(self):
# create camera object
self.camera = pylon.InstantCamera(pylon.TlFactory.GetInstance().CreateDevice(self.device))
# open camera
self.camera.Open()
logger.info(f"Initialized camera {self}")
# load default settings
def _set_defaults(self):
self.camera.UserSetSelector.Value = "Default"
self.camera.UserSetLoad.Execute()
logger.info(f"Loaded default settings for {self}")
# load custom settings
def _setup(self):
# Read the content of the file back to the camera's node map with disabled validation
pylon.FeaturePersistence.Load(self.config, self.camera.GetNodeMap(), False)
logger.info(f"Loaded custom settings for {self}")
# set camera flags
def _post_init(self):
# set frame rate
if FLAGS.fps > 0:
self.camera.AcquisitionFrameRate.Value = FLAGS.fps
self.camera.AcquisitionFrameRateEnable.Value = True
logger.info(f"FPS for {self} is {FLAGS.fps}")
# rotate camera
if FLAGS.rotate:
self.camera.ReverseX.Value = True
self.camera.ReverseY.Value = True
logger.info(f"Rotated camera {self}")
# get pixel format
self.format = self.camera.PixelFormat.Value
logger.info(f"Pixel format for {self} is {self.format}")
# set decompressor
if self.camera.ImageCompressionMode.Value == "BaslerCompressionBeyond":
self.decompressor = pylon.ImageDecompressor()
self.decompressor.SetCompressionDescriptor(self.camera.BslImageCompressionBCBDescriptor.GetAll())
logger.info(f"Compression for {self} is enabled")
def __post_init__(self):
# discover camera
self._pre_init()
# initialize camera
self._init()
# load default settings
self._set_defaults()
# load custom settings
self._setup()
# set camera flags
self._post_init()
class Frames:
def __init__(self, host, key, format):
self.key = key
self.format = format
pool = ConnectionPool(host=host)
self.db = Redis(connection_pool=pool)
self.pool = ThreadPoolExecutor(max_workers=2)
def _add_frame(self, frame, maxlen=10, expiry=5):
try:
# add frame to redis
with self.db.pipeline() as pipe:
pipe.xadd(self.key, {"frame": dumps(frame), "timestamp": time(), "format": self.format}, maxlen=maxlen)
pipe.expire(self.key, expiry)
pipe.execute()
except RedisError as e:
logger.error(f"{self.key}> {e}")
def close(self):
self.db.close()
def __call__(self, frame):
self.pool.submit(self._add_frame, frame)
# self._add_frame(frame)
def discover(model, address, pfs):
cameras = ()
try:
# Get all attached devices and exit application if no device is found
devices = pylon.TlFactory.GetInstance().EnumerateDevices()
# filter devices by model
devices = (dev for dev in devices if dev.GetModelName() == model) if model else devices
# filter devices by address (only for GigE cameras)
devices = (dev for dev in devices if dev.GetDeviceClass() == "BaslerGigE") if address else devices
devices = (dev for dev in devices if dev.GetIpAddress() == address) if address else devices
# create camera objects sorted by serial number
cameras = sorted(Camera(device=dev, config=pfs) for dev in devices)
except pylon.GenericException as e:
logger.error(f"{e}")
return cameras
def grab_loop(key, camera, format, decompressor):
try:
# Grabing Continusely (video) with minimal delay
camera.StartGrabbing(pylon.GrabStrategy_LatestImageOnly)
logger.info(f"Grabbing {key}")
except pylon.GenericException as e:
logger.error(f"{key}> {e}")
camera.Close()
return
# Init redis
f = Frames(FLAGS.redis_host, key, format)
tic, tac = 0, time()
while camera.IsGrabbing():
# Wait for an image and then retrieve it. A timeout of 5s is used.
with camera.RetrieveResult(5 * 1000, pylon.TimeoutHandling_Return) as grab:
# Image grabbed successfully?
if grab.GrabSucceeded():
try:
# Add frame to redis
f(decompressor.DecompressImage(grab).Array if decompressor else grab.Array)
tic += 1
except pylon.GenericException as e:
logger.debug(f"{key}> {e}")
else:
logger.debug(f"{key}> {grab.GetErrorDescription()}")
continue
# LOG
if tic % TIC == 0:
cam_info = f"{key} > fps (app/camera): {(TIC / (time() - tac)):.2f} / {camera.ResultingFrameRate.Value:.2f}"
cam_info += f", payload: {camera.PayloadSize.Value/1024/1024:.2f} MB"
cam_debug = f"{key} > temperature (current/max): {camera.DeviceTemperature.Value:.2f} / {camera.BslTemperatureMax.Value:.2f} °C"
cam_debug += f", status: {camera.BslTemperatureStatus.Value}"
if decompressor:
cam_info += f", compression ratio: {camera.BslImageCompressionLastRatio.Value:.2f} %"
cam_info += f", size: {camera.BslImageCompressionLastSize.Value/1024/1024:.2f} MB"
logger.info(cam_info)
logger.debug(cam_debug)
# reset timer
tac = time()
def main():
# Discover cameras
cameras = discover(model=FLAGS.model, address=FLAGS.address, pfs=FLAGS.pfs)
if not cameras:
logger.error("No cameras found")
return
# Grab Loop
with ThreadPoolExecutor(max_workers=len(cameras)) as exec:
for index, c in enumerate(cameras):
exec.submit(grab_loop, f"{FLAGS.redis_key}:{index}", c.camera, c.format, c.decompressor)
if __name__ == "__main__":
# flags
logger.info(FLAGS)
# run
main()
# close
logger.info("Exit")
Error message:
Run htop and see memory constantly increasing
Is your camera operational in Basler pylon viewer on your platform
Yes
Hardware setup & camera model(s) used
Basler a2A5320-23umPRO (USB3) on Ubuntu 22.04 (arn64) with pypylon==3.0.1
Runtime information:
root@basler-78978b9445-6fnvc:/app# python3
Python 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import sys, pypylon.pylon, platform
>>> print(f'python: {sys.version}')
python: 3.10.12 (main, Nov 20 2023, 15:14:05) [GCC 11.4.0]
>>> print(f'platform: {sys.platform}/{platform.machine()}/{platform.release()}')
platform: linux/aarch64/5.10.160
>>> print(f'pypylon: {pypylon.pylon.__version__} / {".".join([str(i) for i in pypylon.pylon.GetPylonVersion()])}')
pypylon: 3.0.1 / 7.4.0.38864
>>>
About this issue
- Original URL
- State: closed
- Created 3 years ago
- Comments: 17
I have used the following dockerfile (same as you except downloaded tar.gz directly
and i can confirm memory leak is present 😃
I am going to do the following.
and i will get back to you, have a nice weekend.
@thiesmoeller thanks for this. i am going to try. I am actually running it in dockerized container myself, actually it is deployed on k3s (edge kubernetes). I will get back to you and many thanks!
Thanks for your well-prepared report. We will check it and get back to you soon with the result