pyvisa: ValueError after a few hours of communication with a device

This issue is still present in 1.11.4.dev1+g6881af7

It is easily provoked by creating a bad query

In [16]: h.query('SYST:ERR?')
---------------------------------------------------------------------------
RPCError                                  Traceback (most recent call last)
<ipython-input-16-de10d29fd81d> in <module>
----> 1 h.query('SYST:ERR?')

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa/resources/messagebased.py in query(self, message, delay)
    636
    637         """
--> 638         self.write(message)
    639
    640         delay = self.query_delay if delay is None else delay

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa/resources/messagebased.py in write(self, message, termination, encoding)
    195             message += term
    196
--> 197         count = self.write_raw(message.encode(enco))
    198
    199         return count

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa/resources/messagebased.py in write_raw(self, message)
    155
    156         """
--> 157         return self.visalib.write(self.session, message)[0]
    158
    159     def write(

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa_py/highlevel.py in write(self, session, data)
    541         # from the session handle, dispatch to the write method of the session object.
    542         try:
--> 543             written, status_code = self.sessions[session].write(data)
    544         except KeyError:
    545             return 0, self.handle_return_value(session, StatusCode.error_invalid_object)

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa_py/tcpip.py in write(self, data)
    200
    201                 error, size = self.interface.device_write(
--> 202                     self.link, self._io_timeout, self.lock_timeout, flags, block
    203                 )
    204

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa_py/protocols/vxi11.py in device_write(self, link, io_timeout, lock_timeout, flags, data)
    223                 params,
    224                 self.packer.pack_device_write_parms,
--> 225                 self.unpacker.unpack_device_write_resp,
    226             )
    227         except socket.timeout as e:

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa_py/protocols/rpc.py in make_call(self, proc, args, pack_func, unpack_func)
    470         self.timeout += 1.0
    471
--> 472         return super(RawTCPClient, self).make_call(proc, args, pack_func, unpack_func)
    473
    474     def connect(self, timeout=5.0):

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa_py/protocols/rpc.py in make_call(self, proc, args, pack_func, unpack_func)
    236         if pack_func:
    237             pack_func(args)
--> 238         self.do_call()
    239         if unpack_func:
    240             result = unpack_func()

~/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/pyvisa_py/protocols/rpc.py in do_call(self)
    503             # Can't really happen since this is TCP...
    504             msg = "wrong xid in reply {0} instead of {1}"
--> 505             raise RPCError(msg.format(xid, self.lastxid))
    506
    507

RPCError: wrong xid in reply 15 instead of 17

~/tools$ python -m visa info
/home/valentinem/.local/share/virtualenvs/tools-BgNbJulN/lib/python3.6/site-packages/visa.py:23: FutureWarning: The visa module provided by PyVISA is being deprecated. You can replace `import visa` by `import pyvisa as visa` to achieve the same effect.

The reason for the deprecation is the possible conflict with the visa package provided by the https://github.com/visa-sdk/visa-python which can result in hard to debug situations.
  FutureWarning,
Machine Details:
   Platform ID:    Linux-4.15.0-123-generic-x86_64-with-Ubuntu-18.04-bionic
   Processor:      x86_64

Python:
   Implementation: CPython
   Executable:     /home/valentinem/.local/share/virtualenvs/tools-BgNbJulN/bin/python
   Version:        3.6.9
   Compiler:       GCC 8.4.0
   Bits:           64bit
   Build:          Jul 17 2020 12:50:27 (#default)
   Unicode:        UCS4

PyVISA Version: 1.11.4.dev1+g6881af7

Backends:
   ivi:
      Version: 1.11.4.dev1+g6881af7 (bundled with PyVISA)
      Binary library: Not found
   py:
      Version: 0.5.0
      ASRL INSTR: Available via PySerial (3.4)
      USB INSTR: Available via PyUSB (1.1.0). Backend: libusb1
      USB RAW: Available via PyUSB (1.1.0). Backend: libusb1
      TCPIP INSTR: Available
      TCPIP SOCKET: Available
      GPIB INSTR:
         Please install linux-gpib (Linux) or gpib-ctypes (Windows, Linux) to use this resource type. Note that installing gpib-ctypes will give you access to a broader range of funcionality.
         No module named 'gpib'

Does anybody found the solution to the above problem? I am controlling an instrument via pyvisa and facing this value error randomly anytime. I tried to handle the exception but it still crashes my program.