s = boto3.session.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name)
session = s.resource('sqs')
queue = sqs.get_queue_by_name(QueueName=queue_name)
while True:
messages = queue.receive_messages(MaxNumberOfMessages=10, AttributeNames=['All'], MessageAttributeNames=['All'])
if len(messages) == 0:
break
for message in messages:
save_message(message)
message.delete()
Stack Trace
Exception in thread Thread-4:
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 810, in __bootstrap_inner
self.run()
File "/usr/lib/python2.7/threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "/ws/amazon/bin/sqs", line 59, in each_thread
message.delete()
File "/usr/local/lib/python2.7/dist-packages/boto3/resources/factory.py", line 455, in do_action
response = action(self, *args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/boto3/resources/action.py", line 79, in __call__
response = getattr(parent.meta.client, operation_name)(**params)
File "/usr/local/lib/python2.7/dist-packages/botocore/client.py", line 310, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/usr/local/lib/python2.7/dist-packages/botocore/client.py", line 384, in _make_api_call
operation_model, request_dict)
File "/usr/local/lib/python2.7/dist-packages/botocore/endpoint.py", line 111, in make_request
return self._send_request(request_dict, operation_model)
File "/usr/local/lib/python2.7/dist-packages/botocore/endpoint.py", line 140, in _send_request
success_response, exception):
File "/usr/local/lib/python2.7/dist-packages/botocore/endpoint.py", line 213, in _needs_retry
caught_exception=caught_exception)
File "/usr/local/lib/python2.7/dist-packages/botocore/hooks.py", line 226, in emit
return self._emit(event_name, kwargs)
File "/usr/local/lib/python2.7/dist-packages/botocore/hooks.py", line 209, in _emit
response = handler(**kwargs)
File "/usr/local/lib/python2.7/dist-packages/botocore/retryhandler.py", line 183, in __call__
if self._checker(attempts, response, caught_exception):
File "/usr/local/lib/python2.7/dist-packages/botocore/retryhandler.py", line 250, in __call__
caught_exception)
File "/usr/local/lib/python2.7/dist-packages/botocore/retryhandler.py", line 265, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/usr/local/lib/python2.7/dist-packages/botocore/retryhandler.py", line 313, in __call__
caught_exception)
File "/usr/local/lib/python2.7/dist-packages/botocore/retryhandler.py", line 222, in __call__
return self._check_caught_exception(attempt_number, caught_exception)
File "/usr/local/lib/python2.7/dist-packages/botocore/retryhandler.py", line 355, in _check_caught_exception
raise caught_exception
ValueError: filedescriptor out of range in select()
This needs to be reopened. We regularly have more than 1024 sockets open and using boto to upload a file to S3 fails at that state since your vendored
urllib3version usesselect.select().I am having the exact same issue today. I cannot see why it was closed, is there a duplicate issue?
This issue has not been fixed; I think updating the vendored version of urllib3 would do the trick, but there have been enough changes to urllib since version 1.10.x that naively replacing the whole package with the latest version breaks things.