kombu: python exceptions are not json serializable.
I have some celery tasks, which may raise exception, if required. The problem is python Exception object is not json-serialization, yet though yaml can serialize python Exception objects but I don’t have control to change this configuration for celery.
Can we in celery(more specifically in kombu) internally handle, serialization of exception object ?
Here is a snippet which will generate error.
from kombu import Connection
ex = None
try:
a = 1/0
except ZeroDivisionError, e:
ex = e
print ex
conn = Connection('amqp://guest:guest@localhost:5672//')
simple_queue = conn.SimpleQueue('simple_queue')
# No error as expected
simple_queue.put('Hello')
# No error as expected
simple_queue.put(str(ex))
# json cannot serliaze exception
# hence celery gives error
simple_queue.put(ex)
Here is error which I get.
---------------------------------------------------------------------------
EncodeError Traceback (most recent call last)
<ipython-input-22-9667148d81ba> in <module>()
----> 1 sq.put(ex)
/usr/local/lib/python2.7/dist-packages/kombu/simple.pyc in put(self, message, serializer, headers, compression, routing_key, **kwargs)
70 headers=headers,
71 compression=compression,
---> 72 **kwargs)
73
74 def clear(self):
/usr/local/lib/python2.7/dist-packages/kombu/messaging.pyc in publish(self, body, routing_key, delivery_mode, mandatory, immediate, priority, content_type, content_encoding, serializer, headers, compression, exchange, retry, retry_policy, declare, expiration, **properties)
163 body, content_type, content_encoding = self._prepare(
164 body, serializer, content_type, content_encoding,
--> 165 compression, headers)
166
167 publish = self._publish
/usr/local/lib/python2.7/dist-packages/kombu/messaging.pyc in _prepare(self, body, serializer, content_type, content_encoding, compression, headers)
239 serializer = serializer or self.serializer
240 (content_type, content_encoding,
--> 241 body) = dumps(body, serializer=serializer)
242 else:
243 # If the programmer doesn't want us to serialize,
/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in dumps(self, data, serializer)
162
163 with _reraise_errors(EncodeError):
--> 164 payload = encoder(data)
165 return content_type, content_encoding, payload
166 encode = dumps # XXX compat
/usr/lib/python2.7/contextlib.pyc in __exit__(self, type, value, traceback)
33 value = type()
34 try:
---> 35 self.gen.throw(type, value, traceback)
36 raise RuntimeError("generator didn't stop after throw()")
37 except StopIteration, exc:
/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in _reraise_errors(wrapper, include, exclude)
57 raise
58 except include as exc:
---> 59 reraise(wrapper, wrapper(exc), sys.exc_info()[2])
60
61
/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in _reraise_errors(wrapper, include, exclude)
53 include=(Exception, ), exclude=(SerializerNotInstalled, )):
54 try:
---> 55 yield
56 except exclude:
57 raise
/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in dumps(self, data, serializer)
162
163 with _reraise_errors(EncodeError):
--> 164 payload = encoder(data)
165 return content_type, content_encoding, payload
166 encode = dumps # XXX compat
/usr/local/lib/python2.7/dist-packages/anyjson/__init__.pyc in dumps(value)
139 def dumps(value):
140 """Deserialize JSON-encoded object to a Python object."""
--> 141 return implementation.dumps(value)
142 serialize = dumps
/usr/local/lib/python2.7/dist-packages/anyjson/__init__.pyc in dumps(self, data)
85 TypeError if the object could not be serialized."""
86 try:
---> 87 return self._encode(data)
88 except self._encode_error, exc:
89 raise TypeError, TypeError(*exc.args), sys.exc_info()[2]
/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/__init__.pyc in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, encoding, default, use_decimal, namedtuple_as_object, tuple_as_array, bigint_as_string, sort_keys, item_sort_key, for_json, ignore_nan, int_as_string_bitcount, iterable_as_array, **kw)
378 and not kw
379 ):
--> 380 return _default_encoder.encode(obj)
381 if cls is None:
382 cls = JSONEncoder
/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/encoder.pyc in encode(self, o)
273 # exceptions aren't as detailed. The list call should be roughly
274 # equivalent to the PySequence_Fast that ''.join() would do.
--> 275 chunks = self.iterencode(o, _one_shot=True)
276 if not isinstance(chunks, (list, tuple)):
277 chunks = list(chunks)
/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/encoder.pyc in iterencode(self, o, _one_shot)
355 self.iterable_as_array, Decimal=decimal.Decimal)
356 try:
--> 357 return _iterencode(o, 0)
358 finally:
359 key_memo.clear()
/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/encoder.pyc in default(self, o)
250
251 """
--> 252 raise TypeError(repr(o) + " is not JSON serializable")
253
254 def encode(self, o):
About this issue
- Original URL
- State: closed
- Created 8 years ago
- Comments: 15 (8 by maintainers)
Hey guys I guess this should work good enough for any Exception which is not serializable
This worked for me atleast.
@humitos you can use
self.backend.store_result
, passing the exception instance, as seen here. This is a different use case, so please open a separate issue, so that we have better tracking and knowledge base. Let me know if this works!You’re likely to
raise
a custom exception instance which isn’t subclassed fromException
.