vt-py: JSON parse error during iteration
I can’t figure out if this is a problem with VirusTotal API data that is returned, or the vt-py client. I am using vt-py to automate processing of livehunt notifications. It seems to work most of the time, but intermittently, I have been getting the following exception in my logs, and I can’t figure out what the problem is:
2021-06-24 05:37:26 job_scheduler.log [ERROR]::job_scheduler->check_vthunt:314 An exception occured when trying to process vthunt
Traceback (most recent call last):
File "/opt/ems/job_scheduler.py", line 279, in check_vthunt
for notification in notifications:
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 137, in __iter__
self._items, self._server_cursor = self._get_batch()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 114, in _get_batch
self._path, params=self._build_params())
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 403, in get_json
return _make_sync(self.get_json_async(path, *path_args, params=params))
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 53, in _make_sync
return event_loop.run_until_complete(future)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/asyncio/base_events.py", line 488, in run_until_complete
return future.result()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 408, in get_json_async
return await self._response_to_json(response)
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 236, in _response_to_json
return await response.json_async()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 99, in json_async
return await self._aiohttp_resp.json()
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/site-packages/aiohttp/client_reqrep.py", line 1032, in json
return loads(self._body.decode(encoding)) # type: ignore
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 355, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Unterminated string starting at: line 14378 column 35 (char 792013)
2021-06-24 13:38:18 job_scheduler.log [ERROR]::job_scheduler->check_vthunt:314 An exception occured when trying to process vthunt
Traceback (most recent call last):
File "/opt/ems/job_scheduler.py", line 279, in check_vthunt
for notification in notifications:
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 137, in __iter__
self._items, self._server_cursor = self._get_batch()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 114, in _get_batch
self._path, params=self._build_params())
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 403, in get_json
return _make_sync(self.get_json_async(path, *path_args, params=params))
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 53, in _make_sync
return event_loop.run_until_complete(future)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/asyncio/base_events.py", line 488, in run_until_complete
return future.result()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 408, in get_json_async
return await self._response_to_json(response)
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 236, in _response_to_json
return await response.json_async()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 99, in json_async
return await self._aiohttp_resp.json()
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/site-packages/aiohttp/client_reqrep.py", line 1032, in json
return loads(self._body.decode(encoding)) # type: ignore
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 355, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Unterminated string starting at: line 15493 column 33 (char 869532)
2021-06-25 03:00:12 job_scheduler.log [ERROR]::job_scheduler->check_vthunt:314 An exception occured when trying to process vthunt
Traceback (most recent call last):
File "/opt/ems/job_scheduler.py", line 279, in check_vthunt
for notification in notifications:
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 137, in __iter__
self._items, self._server_cursor = self._get_batch()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 114, in _get_batch
self._path, params=self._build_params())
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 403, in get_json
return _make_sync(self.get_json_async(path, *path_args, params=params))
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 53, in _make_sync
return event_loop.run_until_complete(future)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/asyncio/base_events.py", line 488, in run_until_complete
return future.result()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 408, in get_json_async
return await self._response_to_json(response)
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 236, in _response_to_json
return await response.json_async()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 99, in json_async
return await self._aiohttp_resp.json()
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/site-packages/aiohttp/client_reqrep.py", line 1032, in json
return loads(self._body.decode(encoding)) # type: ignore
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 355, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Unterminated string starting at: line 17004 column 17 (char 956831)
2021-06-24 03:37:15 job_scheduler.log [ERROR]::job_scheduler->check_vthunt:314 An exception occured when trying to process vthunt
Traceback (most recent call last):
File "/opt/ems/job_scheduler.py", line 279, in check_vthunt
for notification in notifications:
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 137, in __iter__
self._items, self._server_cursor = self._get_batch()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/iterator.py", line 114, in _get_batch
self._path, params=self._build_params())
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 403, in get_json
return _make_sync(self.get_json_async(path, *path_args, params=params))
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 53, in _make_sync
return event_loop.run_until_complete(future)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/asyncio/base_events.py", line 488, in run_until_complete
return future.result()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 408, in get_json_async
return await self._response_to_json(response)
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 236, in _response_to_json
return await response.json_async()
File "/opt/rh/rh-python36/root/usr/lib/python3.6/site-packages/vt/client.py", line 99, in json_async
return await self._aiohttp_resp.json()
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/site-packages/aiohttp/client_reqrep.py", line 1032, in json
return loads(self._body.decode(encoding)) # type: ignore
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/__init__.py", line 354, in loads
return _default_decoder.decode(s)
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/opt/rh/rh-python36/root/usr/lib64/python3.6/json/decoder.py", line 355, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Unterminated string starting at: line 14384 column 40 (char 792288)
A snippet of my code looks like the following:
with Client(apikey=item.get('api_key'), host=self.vt_host) as client:
notifications = client.iterator('/intelligence/hunting_notification_files', batch_size=10, limit=self.vthunt_limit)
for notification in notifications:
# Process notification...
About this issue
- Original URL
- State: closed
- Created 3 years ago
- Comments: 20 (20 by maintainers)
Hi @mgmacias95 , I made 2 comments on your PR.
Please review them before I test for you.
I just added this, and hopefully it will help me discover the root cause. I will let you know what happens when it fails next time.