spikeinterface: Help me understand why my unit locations don't make sense

Hello and happy new year!

Let me preface my problem by saying that I am not a neuroscientist and do not have a background in signal processing. I am a bioinformatician trying to set up a pipeline for my lab.

I am using spike interface 0.101.2 to interface between Maxwell Two data and spyking-circus 1 & 2.

After sorting (with either version of spyking circus) and creating the analyzer, I want to assess the locations of the sorted neurons on the sensor array. However, all the neurons (up to a 1000 per well) are plotted right on top of each other in just one location or at most in a handful of locations. This happens with both the built-in report, and with custom export and plot, with both sorter versions, with two of the three localisation methods (the third doesn’t put them so much on top of each other, but their distribution is still suspicious and unlikely), through two entirely independent pipelines made by separate people, and for all the wells across multiple plates.

This can’t be right. Even physically impossible. I’d expect neurons to be spread out into a single-layer network over the array. Microscopy and the spread of selected electrodes suggests that they are indeed spread out, and we do get normal looking recordings. So the inferred unit locations do not represent the cell cultures. The question is why.

Am I executing it wrong? Are my parameters way off? Is the sorter unable to sort network bursts? Something else? I feel completely out of my depth…

I am including my code to run with spyking circus 2, omitting some parts that deal with file logistics. Any help is appreciated!

#!/user/bin/env python

### Developed with python 3.12.8

import os
#import sys
import subprocess
import argparse
import re
import numpy as np

import spikeinterface as si
import spikeinterface.extractors as sixtr
#import spikeinterface.preprocessing as sipre
import spikeinterface.sorters as siso
import spikeinterface.qualitymetrics as siqm
import spikeinterface.exporters as sixpo

#import generate_mapping as mgm      ## the function throws an error if used from import




def main(args):

	## Options
	parser = argparse.ArgumentParser(description="Extract spike-sorted data.")
	parser.add_argument("--dataPath", required=False, type=str, default="/SCRATCH/mea/data/", help="Base path for raw recordings.")
	parser.add_argument("--experiment", required=True, type=str, help="Experiment name (== data subdirectory).")
	parser.add_argument("--measurement", required=True, type=str, help="Usually date & condition.")
	parser.add_argument("--plate", required=True, type=str, help="Plate ID.")
	parser.add_argument("--intermediatePath", required=False, type=str, default="Network/", help="Fixed sub-dir in the plate.")
	parser.add_argument("--recording", required=True, type=str, help="6-digit recording id.")
	parser.add_argument("--sortedPath", required=False, type=str, default="/SCRATCH/mea/sorted/", help="Base path for sorted recordings.")
	parser.add_argument("--analysisPath", required=False, type=str, default="/SCRATCH/mea/analysis/", help="Base path for analysis results.")

	parser.add_argument("--numofWells", required=False, type=int, default=None, help="6 or 24.")
	parser.add_argument("--subsample", required=False, type=int, default=1000, help="Number of units to subset for calculations.")

	parser.add_argument("--gm_path", required=False, type=str, default='/BIOINFORMATICS/GH_repos/MEA_ghrepo/generate_mapping.py', help="Maxwell's well map script.")
	parser.add_argument("--hdf5", required=False, type=str, default='/SCRATCH/REFERENCES/maxwell_plugin', help="Maxwell HDF5 plugin.") # /BIOINFORMATICS/GH_repos/MEA_ghrepo/maxwell_plugin

	parser.add_argument("--cores", required=False, type=int, default=1, help="CPU cores.")
	parser.add_argument("--memory", required=False, type=str, default='280G', help="RAM.")
	

	# Hard-code some values for easy interactive development
	args = ['sort.py',
		'--experiment', 'DoseRes',
		'--measurement', '240910_7W',
		'--plate', 'M07100',
		'--recording', '000022'
	]

	params = parser.parse_args(args[1:])

	os.environ['HDF5_PLUGIN_PATH'] = params.hdf5
	global_job_kwargs = dict(n_jobs = params.cores, total_memory = params.memory, mp_context = 'spawn', progress_bar = False)
	si.set_global_job_kwargs(**global_job_kwargs)


## omitting logistics ##

	wells = ['000', '001', '003', '004', '005'] 
	well_ids = {WELL: '_'.join([params.plate, 'well' + WELL, params.measurement, params.recording]) for WELL in wells
					for WELL in wells }

	recordings = {WELL: sixtr.read_maxwell(file_path=os.path.join(rec_path, 'data.raw.h5'), stream_id = 'well' + WELL, rec_name = 'rec0000', install_maxwell_plugin = False) 
					for WELL in wells }



	## Prepare to sort the whole plate with the internal SpykingCircus2

	joblist = [
		{
			'sorter_name': 'spykingcircus2', 
			'recording': recordings[wells[i]], 
			'output_folder': os.path.join(params.sortedPath, params.experiment, well_ids[wells[i]]),
			'remove_existing_folder' : True,
			'verbose' : False,
			'raise_error' : False,

			'general' : {'ms_before': 2, 'ms_after': 2, 'radius_um': 100}, 
			'sparsity' : {'method': 'snr', 'amplitude_mode': 'peak_to_peak', 'threshold': 0.25}, 
			'filtering' : {'freq_min': 150, 'freq_max': (recordings[wells[i]].sampling_frequency / 2)-1, 'ftype': 'bessel', 'filter_order': 2}, 
			'detection' : {'peak_sign': 'neg', 'detect_threshold': 4}, 
			'selection' : {'method': 'uniform', 'n_peaks_per_channel': 5000, 'min_n_peaks': 100000, 'select_per_channel': False, 'seed': 42}, 
			'apply_motion_correction' : True, 
			'motion_correction' : {'preset': 'nonrigid_fast_and_accurate'}, 
			'merging' : {'similarity_kwargs': {'method': 'cosine', 'support': 'union', 'max_lag_ms': 0.2}, 
					'correlograms_kwargs': {}, 
					'auto_merge': {'min_spikes': 10, 'corr_diff_thresh': 0.25}}, 
			'clustering' : {'legacy': True}, 
			'matching' : {'method': 'wobble'}, 
			'apply_preprocessing' : True, 
			'matched_filtering' : True, 
			'cache_preprocessing' : {'mode': 'memory', 'memory_limit': 0.9, 'delete_cache': True}, 
			'multi_units_only' : False, 
			'job_kwargs' : {'n_jobs': 1}, 
			'debug' : True
		}
		for i in list(range(0, len(wells)))
	]




	## Sort in parallel
	sortings = siso.run_sorter_jobs(job_list=joblist, engine='joblib', return_output = False)  

	del(recordings) ## From here on deal with one at a time
	del(sortings)





	## Post-process and extract

	overview = []  ## collect some result metrics

	## Basic overview

	for WELL in wells:
		w = well_ids[WELL]
		print(w)
		
		## Import one pair unsorted/sorted data
		recording = sixtr.read_maxwell(file_path=os.path.join(rec_path, 'data.raw.h5'), stream_id = 'well' + WELL, rec_name = 'rec0000', install_maxwell_plugin = False) 
		sorting = si.read_numpy_sorting_folder(os.path.join(params.sortedPath, params.experiment, w, 'sorter_output', 'sorting'))
		sorting.register_recording(recording)
		
		## Some basic metrics
		print('Import')
		overview.append([params.experiment,
							w,
							sorting.get_sampling_frequency(),
							sorting.get_total_duration(),
							sorting.get_num_units(),
							len(sorting.get_empty_unit_ids()),
							sorting.get_num_samples(),
							sorting.get_num_segments()])
		
		## Curate?

		#sorting_dedup = scur.remove_duplicated_spikes(sorting, censored_period_ms=0.1, method = 'keep_first_iterative' )     ## not sure what value of time would be good in the context of neuron network bursts
		#sorting_95 = scur.remove_redundant_units(sorting, duplicate_threshold = 0.95, remove_strategy='max_spikes', align = False)     ## not sure we want to use this in the context of neuron network bursts
		sorting = sorting.remove_empty_units()

		## Combine

		print('Analyzer')
		#analyzer = si.load_sorting_analyzer(folder = os.path.join(params.analysisPath, params.experiment, w + '_analyzer.zarr'))
		#analyzer = si.create_sorting_analyzer(sorting = sorting, recording = recording, format = 'memory')
		analyzer = si.create_sorting_analyzer(sorting = sorting, 
											recording = recording, 
											folder = os.path.join(params.analysisPath, params.experiment, w + '_analyzer'),
											format = 'zarr',
											overwrite = True)
		
		## omitting various info extractions ##


		analyzer.compute("random_spikes", method="uniform", max_spikes_per_unit = params.subsample)
		analyzer.compute("waveforms")
		analyzer.compute("templates")
		
		## Extract neuron locations -----
		ul = analyzer.compute("unit_locations", method="center_of_mass")
		uloc = ul.get_data()
		uids = ul.sorting_analyzer.unit_ids
		uids = np.array(list(zip(uids, ['unit'] * len(uids))))
		uloc = np.hstack((uids, uloc))
		np.savetxt(os.path.join(params.analysisPath, params.experiment, w + '_unit_locations.tsv'), 
					uloc, 
					header = "\t".join(['ID', 'Type', 'X', 'Y', 'Z']),  ## Z not available from all methods. And in any case irrelevant for a 2D array. Having an extra column header is easier to handle than it missing.
					fmt = '%s', delimiter = "\t")
		
		analyzer.compute("spike_amplitudes")
		analyzer.compute("noise_levels")
		analyzer.compute("isi_histograms", window_ms = 100, bin_ms = 5)
		analyzer.compute("correlograms", window_ms=100, bin_ms=5)
		analyzer.compute("template_metrics", include_multi_channel_metrics = False,
								metric_kwargs = {"recovery_window_ms" : 0.7, "peak_relative_threshold" : 0.2, "peak_width_ms" : 0.2,
												"min_channels_for_velocity" : 5, "min_r2_velocity" : 0.7, "exp_peak_function" : "ptp",
												"min_r2_exp_decay" : 0.5, "spread_threshold" : 0.2, "spread_smooth_um" : 20})
		analyzer.compute("principal_components", n_components=3, mode="by_channel_local", whiten=True) 
		try:
			analyzer.compute("amplitude_scalings") 
		#except LinAlgError:     ## Don't know where LinAlgError is defined, clearly not in my current imports.
		except: 
			raise Warning("An error, possibly LinAlgError due to expected matrix being singular, occurred. Computation of Amplitude Scalings aborted.")
			## I have no idea how to prevent it or how to fix it. I don't even know if it is a useful metric for us. For now I'm passing the ball downstream, to the interpretation of any metrics.
		analyzer.compute("spike_locations", ms_before = 0.5, method = "monopolar_triangulation")
		analyzer.compute("template_similarity", method = "cosine", max_lag_ms = 0, support = "union")
		analyzer.compute("quality_metrics", skip_pc_metrics = False)
		

		
		## Extract analyzer metrics
		sixpo.export_report(analyzer, os.path.join(params.analysisPath, params.experiment, w + '_sorting_report'),
						remove_if_exists=True, format='pdf', show_figures=False)





if __name__ == '__main__':
    sys.exit(main())

About this issue

  • Original URL
  • State: closed
  • Created 2 years ago
  • Comments: 18 (7 by maintainers)

Most upvoted comments

I believe you guys nailed it with the unsorted/sorted mismatch! Things are looking up!

I’ve added this pre-processing step (please, feel free to recommend improvements, I am still not a signal specialist):

recording = sipre.bandpass_filter(recording, freq_min = 100.0, freq_max = 3500, margin_ms = 5.0, dtype = None)

bad_ids = sipre.detect_bad_channels(recording, method = 'coherence+psd', std_mad_threshold = 5, psd_hf_threshold = 0.02, dead_channel_threshold = -0.5, noisy_channel_threshold = 1.0, outside_channel_threshold = -0.75, outside_channels_location = 'top', n_neighbors = 11, nyquist_threshold = 0.8, direction = 'y', chunk_duration_s = 0.3, num_random_chunks = 100, welch_window_ms = 10.0, neighborhood_r2_threshold = 0.9, neighborhood_r2_radius_um = 30.0, seed = None)

recording.remove_channels(bad_ids)	
	
recording = sipre.correct_motion(recording, preset = 'dredge_fast', folder = None, output_motion = False, output_motion_info = False)

and changed my sorting prameters to this (again, suggestions are welcome):

joblist = [
	{
		'sorter_name': 'spykingcircus2', 
		'recording': recordings[wells[i]], 
		'output_folder': os.path.join(params.sortedPath, params.experiment, 'spykingcircus2', well_ids[wells[i]]),
		'remove_existing_folder' : True,
		'verbose' : False,
		'raise_error' : False,
		'general' : {'ms_before': 2, 'ms_after': 2, 'radius_um': 100}, 
		'sparsity' : {'method': 'snr', 'amplitude_mode': 'peak_to_peak', 'threshold': 0.25}, 
		'detection' : {'peak_sign': 'neg', 'detect_threshold': 4}, 
		'selection' : {'method': 'uniform', 'n_peaks_per_channel': 5000, 'min_n_peaks': 10000, 'select_per_channel': False}, 
		'apply_motion_correction' : False, 
		'merging' : {'similarity_kwargs': {'method': 'cosine', 'support': 'union', 'max_lag_ms': 0.2}, 
		'correlograms_kwargs': {}, 
		'auto_merge': {'min_spikes': 10, 'corr_diff_thresh': 0.25}}, 
		'clustering' : {'legacy': True}, 
		'apply_preprocessing' : False, 
		'matched_filtering' : False, 
		'cache_preprocessing' : {'mode': 'memory', 'memory_limit': 0.9, 'delete_cache': True}, 
		'multi_units_only' : False, 
		'job_kwargs' : {'n_jobs': 1}, 
		'debug' : True
	}
	for i in list(range(0, len(wells)))
]

sortings = siso.run_sorter_jobs(job_list=joblist, engine='joblib', return_output = False)

and the localisations are starting to look like this, which looks very plausible:

unit_locations.pdf

Thank you so much for your help!