arrow: [CI][Python] Windows wheels job fail on test_cython_api
Describe the bug, including details regarding any error messages, version, and platform.
The Windows wheels jobs have started failing since a couple of days ago (this where the changes between last success and first failure: https://github.com/apache/arrow/compare/e32f56b478171fc4b53dc2042c4cf5d37c97e351...21703f0d5e140ce28bd64275a7a5595d9a8ae75c)
- wheel-windows-cp310-cp310-amd64
- wheel-windows-cp311-cp311-amd64
- wheel-windows-cp312-cp312-amd64
- wheel-windows-cp313-cp313-amd64
- wheel-windows-cp313-cp313t-amd64
- wheel-windows-cp39-cp39-amd64
The error on CI:
================================== FAILURES ===================================
_______________________________ test_cython_api _______________________________
tmpdir = local('C:\\Users\\ContainerAdministrator\\AppData\\Local\\Temp\\pytest-of-ContainerAdministrator\\pytest-0\\test_cython_api0')
@pytest.mark.numpy
@pytest.mark.cython
def test_cython_api(tmpdir):
"""
Basic test for the Cython API.
"""
# Fail early if cython is not found
import cython # noqa
with tmpdir.as_cwd():
# Set up temporary workspace
pyx_file = 'pyarrow_cython_example.pyx'
shutil.copyfile(os.path.join(here, pyx_file),
os.path.join(str(tmpdir), pyx_file))
# Create setup.py file
setup_code = setup_template.format(pyx_file=pyx_file,
compiler_opts=compiler_opts,
test_ld_path=test_ld_path)
with open('setup.py', 'w') as f:
f.write(setup_code)
# ARROW-2263: Make environment with this pyarrow/ package first on the
# PYTHONPATH, for local dev environments
subprocess_env = test_util.get_modified_env_with_pythonpath()
# Compile extension module
subprocess.check_call([sys.executable, 'setup.py',
'build_ext', '--inplace'],
env=subprocess_env)
# Check basic functionality
orig_path = sys.path[:]
sys.path.insert(0, str(tmpdir))
try:
mod = __import__('pyarrow_cython_example')
check_cython_example_module(mod)
finally:
sys.path = orig_path
# Check the extension module is loadable from a subprocess without
# pyarrow imported first.
code = """if 1:
import sys
import os
try:
# Add dll directory was added on python 3.8
# and is required in order to find extra DLLs
# only for win32
for dir in {library_dirs}:
os.add_dll_directory(dir)
except AttributeError:
pass
mod = __import__({mod_name!r})
arr = mod.make_null_array(5)
assert mod.get_array_length(arr) == 5
assert arr.null_count == 5
""".format(mod_name='pyarrow_cython_example',
library_dirs=pa.get_library_dirs())
path_var = None
if sys.platform == 'win32':
if not hasattr(os, 'add_dll_directory'):
# Python 3.8 onwards don't check extension module DLLs on path
# we have to use os.add_dll_directory instead.
delim, path_var = ';', 'PATH'
elif sys.platform == 'darwin':
delim, path_var = ':', 'DYLD_LIBRARY_PATH'
else:
delim, path_var = ':', 'LD_LIBRARY_PATH'
if path_var:
paths = sys.path
paths += pa.get_library_dirs()
paths += [subprocess_env.get(path_var, '')]
paths = [path for path in paths if path]
subprocess_env[path_var] = delim.join(paths)
> subprocess.check_call([sys.executable, '-c', code],
stdout=subprocess.PIPE,
env=subprocess_env)
Python313\Lib\site-packages\pyarrow\tests\test_cython.py:163:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
popenargs = (['C:\\Python313\\python.exe', '-c', 'if 1:\n import sys\n import os\n\n try:\n ..._null_array(5)\n assert mod.get_array_length(arr) == 5\n assert arr.null_count == 5\n '],)
kwargs = {'env': {'ACTIVE_PERL_HOME': 'C:\\Perl', 'ACTIVE_PERL_URL': 'https://cli-msi.s3.amazonaws.com/', 'ACTIVE_PERL_VERSION': '5.28', 'ALLUSERSPROFILE': 'C:\\ProgramData', ...}, 'stdout': -1}
retcode = 1
cmd = ['C:\\Python313\\python.exe', '-c', 'if 1:\n import sys\n import os\n\n try:\n ...ke_null_array(5)\n assert mod.get_array_length(arr) == 5\n assert arr.null_count == 5\n ']
def check_call(*popenargs, **kwargs):
"""Run command with arguments. Wait for command to complete. If
the exit code was zero then return, otherwise raise
CalledProcessError. The CalledProcessError object will have the
return code in the returncode attribute.
The arguments are the same as for the call function. Example:
check_call(["ls", "-l"])
"""
retcode = call(*popenargs, **kwargs)
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
> raise CalledProcessError(retcode, cmd)
E subprocess.CalledProcessError: Command '['C:\\Python313\\python.exe', '-c', "if 1:\n import sys\n import os\n\n try:\n # Add dll directory was added on python 3.8\n # and is required in order to find extra DLLs\n # only for win32\n for dir in ['C:\\\\Python313\\\\Lib\\\\site-packages\\\\pyarrow']:\n os.add_dll_directory(dir)\n except AttributeError:\n pass\n\n mod = __import__('pyarrow_cython_example')\n arr = mod.make_null_array(5)\n assert mod.get_array_length(arr) == 5\n assert arr.null_count == 5\n "]' returned non-zero exit status 1.
Python313\Lib\subprocess.py:419: CalledProcessError
---------------------------- Captured stdout call -----------------------------
Compiling pyarrow_cython_example.pyx because it changed.
[1/1] Cythonizing pyarrow_cython_example.pyx
Extension module: <setuptools.extension.Extension('pyarrow_cython_example') at 0x1db6b5bf4d0> ['C:\\Python313\\Lib\\site-packages\\numpy\\_core\\include', 'C:\\Python313\\Lib\\site-packages\\pyarrow\\include'] ['arrow_python', 'arrow'] ['C:\\Python313\\Lib\\site-packages\\pyarrow']
running build_ext
building 'pyarrow_cython_example' extension
creating build\temp.win-amd64-cpython-313\Release
"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29333\bin\HostX86\x64\cl.exe" /c /nologo /O2 /W3 /GL /DNDEBUG /MD -IC:\Python313\Lib\site-packages\numpy\_core\include -IC:\Python313\Lib\site-packages\pyarrow\include -IC:\Python313\include -IC:\Python313\Include "-IC:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29333\ATLMFC\include" "-IC:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29333\include" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\ucrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\shared" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\um" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\winrt" "-IC:\Program Files (x86)\Windows Kits\10\include\10.0.18362.0\cppwinrt" /EHsc /Tppyarrow_cython_example.cpp /Fobuild\temp.win-amd64-cpython-313\Release\pyarrow_cython_example.obj -D_ENABLE_EXTENDED_ALIGNED_STORAGE /std:c++17
pyarrow_cython_example.cpp
creating C:\Users\ContainerAdministrator\AppData\Local\Temp\pytest-of-ContainerAdministrator\pytest-0\test_cython_api0\build\lib.win-amd64-cpython-313
"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29333\bin\HostX86\x64\link.exe" /nologo /INCREMENTAL:NO /LTCG /DLL /MANIFEST:EMBED,ID=2 /MANIFESTUAC:NO /LIBPATH:C:\Python313\Lib\site-packages\pyarrow /LIBPATH:C:\Python313\libs /LIBPATH:C:\Python313 /LIBPATH:C:\Python313\PCbuild\amd64 "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29333\ATLMFC\lib\x64" "/LIBPATH:C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.28.29333\lib\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.18362.0\ucrt\x64" "/LIBPATH:C:\Program Files (x86)\Windows Kits\10\lib\10.0.18362.0\um\x64" arrow_python.lib arrow.lib /EXPORT:PyInit_pyarrow_cython_example build\temp.win-amd64-cpython-313\Release\pyarrow_cython_example.obj /OUT:build\lib.win-amd64-cpython-313\pyarrow_cython_example.cp313-win_amd64.pyd /IMPLIB:build\temp.win-amd64-cpython-313\Release\pyarrow_cython_example.cp313-win_amd64.lib
Creating library build\temp.win-amd64-cpython-313\Release\pyarrow_cython_example.cp313-win_amd64.lib and object build\temp.win-amd64-cpython-313\Release\pyarrow_cython_example.cp313-win_amd64.exp
Generating code
Finished generating code
copying build\lib.win-amd64-cpython-313\pyarrow_cython_example.cp313-win_amd64.pyd ->
---------------------------- Captured stderr call -----------------------------
Traceback (most recent call last):
File "<string>", line 14, in <module>
mod = __import__('pyarrow_cython_example')
ImportError: DLL load failed while importing pyarrow_cython_example: The specified module could not be found.
Component(s)
Python, Continuous Integration
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Comments: 17 (17 by maintainers)
A first step for debugging would be to inspect
pa.get_library_dirs().