spikeinterface: Error running run_sorter in py script from command prompt

Hi!

I’m trying to run Ironclust using Spike Interface on a .py file that I want to directly run from the command prompt. However, when I try to do so, I get an odd error likely due to saving of the data to the output folder. When I run the code from jupyter notebook, it runs completely fine, but the error seems to stem from using a .py file. I’ve attached the output of my error message below. If anyone has encountered a similar error before and would know a fix, that’d be great, thanks!

Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 125, in _main
    prepare(preparation_data)
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 236, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
    main_content = runpy.run_path(main_path,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 289, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 96, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "C:\Users\Allison\source\repos\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1.py", line 149, in <module>
    IC_si = ss.run_sorter('ironclust', recording = recording_prb, output_folder = r'C:\Users\Allison\Desktop\auto_pilot_721', verbose=False)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 143, in run_sorter
    return run_sorter_local(**common_kwargs)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 158, in run_sorter_local
    output_folder = SorterClass.initialize_folder(
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\basesorter.py", line 120, in initialize_folder
    shutil.rmtree(str(output_folder))
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 750, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 615, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 615, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 620, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 618, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'C:\\Users\\Allison\\Desktop\\auto_pilot_721\\sorter_output\\ironclust_dataset\\raw.mda'
Running ironclust...
Running ironclust...
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 125, in _main
    prepare(preparation_data)
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 236, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
    main_content = runpy.run_path(main_path,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 289, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 96, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "C:\Users\Allison\source\repos\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1.py", line 149, in <module>
    IC_si = ss.run_sorter('ironclust', recording = recording_prb, output_folder = r'C:\Users\Allison\Desktop\auto_pilot_721', verbose=False)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 143, in run_sorter
    return run_sorter_local(**common_kwargs)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 158, in run_sorter_local
    output_folder = SorterClass.initialize_folder(
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\basesorter.py", line 120, in initialize_folder
    shutil.rmtree(str(output_folder))
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 750, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 615, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 615, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 620, in _rmtree_unsafe
Setting IRONCLUST_PATH environment variable for subprocess calls to: C:\Users\Allison\Desktop\IntanToNWB-main\ironclust
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 618, in _rmtree_unsafe
Traceback (most recent call last):
    os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'C:\\Users\\Allison\\Desktop\\auto_pilot_721\\sorter_output\\ironclust_dataset\\raw.mda'
  File "<string>", line 1, in <module>
C:\Users\Allison\Desktop\burgers_auto.rhd
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 116, in spawn_main
    exitcode = _main(fd, parent_sentinel)
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 125, in _main
    prepare(preparation_data)
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 236, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "C:\ProgramData\miniconda3\lib\multiprocessing\spawn.py", line 287, in _fixup_main_from_path
    main_content = runpy.run_path(main_path,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 289, in run_path
    return _run_module_code(code, init_globals, run_name,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 96, in _run_module_code
    _run_code(code, mod_globals, init_globals,
  File "C:\ProgramData\miniconda3\lib\runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "C:\Users\Allison\source\repos\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1.py", line 149, in <module>
    IC_si = ss.run_sorter('ironclust', recording = recording_prb, output_folder = r'C:\Users\Allison\Desktop\auto_pilot_721', verbose=False)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 143, in run_sorter
    return run_sorter_local(**common_kwargs)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 158, in run_sorter_local
    output_folder = SorterClass.initialize_folder(
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\basesorter.py", line 120, in initialize_folder
    shutil.rmtree(str(output_folder))
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 750, in rmtree
    return _rmtree_unsafe(path, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 615, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 615, in _rmtree_unsafe
    _rmtree_unsafe(fullname, onerror)
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 620, in _rmtree_unsafe
    onerror(os.unlink, fullname, sys.exc_info())
  File "C:\ProgramData\miniconda3\lib\shutil.py", line 618, in _rmtree_unsafe
    os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'C:\\Users\\Allison\\Desktop\\auto_pilot_721\\sorter_output\\ironclust_dataset\\raw.mda'
Channel ids: <bound method BaseRecordingSnippets.get_channel_ids of IntanRecordingExtractor: 128 channels - 1 segments - 30.0kHz - 60.002s
  file_path: C:\Users\Allison\Desktop\burgers_auto.rhd>
Sampling frequency: 30000
Number of channels: 128
Running ironclust...
Traceback (most recent call last):
  File "C:\Users\Allison\source\repos\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1\Spike_Sorting_Automation_1.py", line 149, in <module>
    IC_si = ss.run_sorter('ironclust', recording = recording_prb, output_folder = r'C:\Users\Allison\Desktop\auto_pilot_721', verbose=False)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 143, in run_sorter
    return run_sorter_local(**common_kwargs)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\runsorter.py", line 162, in run_sorter_local
    SorterClass.setup_recording(recording, output_folder, verbose=verbose)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\basesorter.py", line 196, in setup_recording
    cls._setup_recording(recording, sorter_output_folder, sorter_params, verbose)
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\sorters\external\ironclust.py", line 169, in _setup_recording
    MdaRecordingExtractor.write_recording(recording=recording, save_path=str(dataset_dir), verbose=False,
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\extractors\mdaextractors.py", line 118, in write_recording
    write_binary_recording(recording, file_paths=save_file_path, dtype=dtype,
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\core\core_tools.py", line 280, in write_binary_recording
    executor.run()
  File "C:\Users\Allison\AppData\Roaming\Python\Python310\site-packages\spikeinterface\core\job_tools.py", line 364, in run
    for res in results:
  File "C:\ProgramData\miniconda3\lib\concurrent\futures\process.py", line 570, in _chain_from_iterable_of_lists
    for element in iterable:
  File "C:\ProgramData\miniconda3\lib\concurrent\futures\_base.py", line 621, in result_iterator
    yield _result_or_cancel(fs.pop())
  File "C:\ProgramData\miniconda3\lib\concurrent\futures\_base.py", line 319, in _result_or_cancel
    return fut.result(timeout)
  File "C:\ProgramData\miniconda3\lib\concurrent\futures\_base.py", line 458, in result
    return self.__get_result()
  File "C:\ProgramData\miniconda3\lib\concurrent\futures\_base.py", line 403, in __get_result
    raise self._exception
concurrent.futures.process.BrokenProcessPool: A process in the process pool was terminated abruptly while the future was running or pending.

About this issue

Original URL
State: open
Created a year ago
Comments: 44 (2 by maintainers)

Most upvoted comments

I guess there is still the issue or a related one, trying to run built-in spike sorters give an error related to shared memory: ` nbytes = int(np.prod(shape) * dtype.itemsize) Error running tridesclous2 Traceback (most recent call last): File “D:\Users\ggaugain\Documents\postdoc\neuron_recordings\test_spikeinterface.py”, line 104, in sorting = si.run_sorter(sorter_name=sname, recording=recording_saved, output_folder=base_folder /sname, remove_existing_folder=True,verbose=True, **sorter_params) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\sorters\runsorter.py”, line 174, in run_sorter return run_sorter_local(**common_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\sorters\runsorter.py”, line 224, in run_sorter_local SorterClass.run_from_folder(output_folder, raise_error, verbose) File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\sorters\basesorter.py”, line 293, in run_from_folder raise SpikeSortingError( spikeinterface.sorters.utils.misc.SpikeSortingError: Spike sorting error trace: Traceback (most recent call last): File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\sorters\basesorter.py”, line 258, in run_from_folder SorterClass._run_from_folder(sorter_output_folder, sorter_params, verbose) File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\sorters\internal\tridesclous2.py”, line 123, in _run_from_folder recording = cache_preprocessing( ^^^^^^^^^^^^^^^^^^^^ File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\sortingcomponents\tools.py”, line 92, in cache_preprocessing recording = recording.save_to_memory(format=“memory”, shared=True, **job_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\core\base.py”, line 853, in save_to_memory cached = self._save(format=“memory”, sharedmem=sharedmem, **save_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\core\baserecording.py”, line 490, in _save cached = SharedMemoryRecording.from_recording(self, **job_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\core\numpyextractors.py”, line 215, in from_recording traces_list, shms = write_memory_recording(source_recording, buffer_type=“sharedmem”, **job_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\core\recording_tools.py”, line 322, in write_memory_recording arr, shm = make_shared_array(shape, dtype) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\site-packages\spikeinterface\core\core_tools.py”, line 167, in make_shared_array shm = SharedMemory(name=None, create=True, size=nbytes) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

File “D:\Users\ggaugain\Anaconda3\envs\si_env\Lib\multiprocessing\shared_memory.py”, line 77, in init raise ValueError(“‘size’ must be a positive integer”)

ValueError: ‘size’ must be a positive integer`

This error I actually know how to solve. It is caused by that np.prod.

I will make a patch.

h-mayorquin on Mar 22, 2024