linxi-tridesclous 1.0.0__tar.gz

linxi_tridesclous-1.0.0/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 tridesclous
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

linxi_tridesclous-1.0.0/MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE

linxi_tridesclous-1.0.0/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: linxi-tridesclous
+Version: 1.0.0
+Summary: offline/online spike sorting with french touch that light the barbecue
+License: MIT
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: pandas
+Requires-Dist: openpyxl
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: neo
+Requires-Dist: tqdm
+Requires-Dist: pyqtgraph
+Requires-Dist: joblib
+Requires-Dist: numba
+Requires-Dist: hdbscan
+Requires-Dist: loky
+Provides-Extra: online
+Requires-Dist: pyacq; extra == "online"
+Provides-Extra: opencl
+Requires-Dist: pyopencl; extra == "opencl"
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary

linxi_tridesclous-1.0.0/README.md

@@ -0,0 +1,73 @@
+# trisdesclous: spike sorting with a [French touch](https://fr.wikipedia.org/wiki/French_touch_(informatique)).
+
+
+**trisdesclous is not maintenaned anymore, please user spikeinterface and treidesclous2 instead**
+
+
+
+![icon](tridesclous/gui/icons/png/main_icon.png)
+
+Authors: Christophe Pouzat and Samuel Garcia
+
+## documentation
+
+The documentation is here: http://tridesclous.readthedocs.io/
+
+## overview
+
+**tris des clous** is a very dishonest translation of **spike sorting** to French.
+
+Pronouce it [tree day clue] in English.
+
+The primary goal of tridesclous was to provide a toolkit to teach good practices in spike sorting techniques.
+Trideslcous is now mature and can be used to sort spike on tetrode up to neuropixel probe recorded dataset.
+
+Tridesclous is both:
+
+  * an offline spike sorter tools able to read many file format
+  * a realtime spike sorting combined with [pyacq](http://pyacq.readthedocs.io)
+
+## Main features
+
+  * template matching based method
+  * several alternative methods at several processing steps of the chain
+  * Qt GUI for interactive exploration
+  * a simple python API to use in a notebook or to build a custom pipeline
+  * support for many data formats (Raw, Blackrock, Neuralynx, Plexon, Spike2, Tdt, OpenEphys, ...) via [neo](https://github.com/NeuralEnsemble/python-neo)
+  * hardware acceleration with opencl: both GPU and multicore CPU
+  * smaller memory footprint
+  * built-in sample datasets to try on
+  * quite fast for tetrode datasets -- expect x30 speedup over real time on a simple laptop
+  * multi-platform
+  * open source based on a true open source stack
+
+The forest of spike sorting tools is dense and *tridesclous* is a new tree.
+Be curious and try it.
+
+
+# Installation
+
+http://tridesclous.readthedocs.io/en/latest/installation.html
+
+# Launch
+
+http://tridesclous.readthedocs.io/en/latest/launch.html
+
+
+## Screenshots
+
+### Offline Catalogue Window
+![snapshot](doc/img/snapshot_cataloguewindow.png)
+
+### offline Peeler Window
+![snapshot](doc/img/snapshot_peelerwindow.png)
+
+### Online Peeler in a pyacq.Node
+![snapshot](doc/img/online_tridesclous.gif)
+
+
+## Status
+
+[![readthedocs](https://readthedocs.org/projects/tridesclous/badge/?version=latest&style=flat)]( http://tridesclous.readthedocs.io/en/latest/?badge=latest)
+[![CircleCI](https://circleci.com/gh/tridesclous/tridesclous.svg?style=svg)](https://circleci.com/gh/tridesclous/tridesclous)
+[![Appveyor](https://ci.appveyor.com/api/projects/status/7cqmevwu0r3sq87e?svg=true)](https://ci.appveyor.com/project/samuelgarcia/tridesclous)

linxi_tridesclous-1.0.0/linxi_tridesclous.egg-info/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: linxi-tridesclous
+Version: 1.0.0
+Summary: offline/online spike sorting with french touch that light the barbecue
+License: MIT
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: pandas
+Requires-Dist: openpyxl
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: neo
+Requires-Dist: tqdm
+Requires-Dist: pyqtgraph
+Requires-Dist: joblib
+Requires-Dist: numba
+Requires-Dist: hdbscan
+Requires-Dist: loky
+Provides-Extra: online
+Requires-Dist: pyacq; extra == "online"
+Provides-Extra: opencl
+Requires-Dist: pyopencl; extra == "opencl"
+Dynamic: license
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary

linxi_tridesclous-1.0.0/linxi_tridesclous.egg-info/SOURCES.txt

@@ -0,0 +1,114 @@
+LICENSE
+MANIFEST.in
+README.md
+setup.py
+linxi_tridesclous.egg-info/PKG-INFO
+linxi_tridesclous.egg-info/SOURCES.txt
+linxi_tridesclous.egg-info/dependency_links.txt
+linxi_tridesclous.egg-info/entry_points.txt
+linxi_tridesclous.egg-info/requires.txt
+linxi_tridesclous.egg-info/top_level.txt
+tridesclous/__init__.py
+tridesclous/autoparams.py
+tridesclous/catalogueconstructor.py
+tridesclous/cataloguetools.py
+tridesclous/cleancluster.py
+tridesclous/cltools.py
+tridesclous/cluster.py
+tridesclous/dataio.py
+tridesclous/datasets.py
+tridesclous/datasource.py
+tridesclous/decomposition.py
+tridesclous/dip.py
+tridesclous/export.py
+tridesclous/importers.py
+tridesclous/iotools.py
+tridesclous/jobtools.py
+tridesclous/labelcodes.py
+tridesclous/matplotlibplot.py
+tridesclous/metrics.py
+tridesclous/numba_tools.py
+tridesclous/peakdetector.py
+tridesclous/peeler.py
+tridesclous/peeler_cl.py
+tridesclous/peeler_engine_base.py
+tridesclous/peeler_engine_geometry.py
+tridesclous/peeler_engine_geometry_cl.py
+tridesclous/peeler_engine_testing.py
+tridesclous/peeler_tools.py
+tridesclous/probe_list.py
+tridesclous/pruningshears.py
+tridesclous/pythran_tools.py
+tridesclous/report.py
+tridesclous/sawchaincut.py
+tridesclous/signalpreprocessor.py
+tridesclous/tools.py
+tridesclous/version.py
+tridesclous/waveformtools.py
+tridesclous/gui/__init__.py
+tridesclous/gui/base.py
+tridesclous/gui/baselist.py
+tridesclous/gui/cataloguecontroller.py
+tridesclous/gui/cataloguewindow.py
+tridesclous/gui/crosscorrelogramviewer.py
+tridesclous/gui/featuretimeviewer.py
+tridesclous/gui/gpuselector.py
+tridesclous/gui/gui_params.py
+tridesclous/gui/initializedatasetwindow.py
+tridesclous/gui/isiviewer.py
+tridesclous/gui/mainwindow.py
+tridesclous/gui/myqt.py
+tridesclous/gui/ndscatter.py
+tridesclous/gui/pairlist.py
+tridesclous/gui/peaklists.py
+tridesclous/gui/peelercontroller.py
+tridesclous/gui/peelerwindow.py
+tridesclous/gui/probegeometryview.py
+tridesclous/gui/silhouette.py
+tridesclous/gui/similarity.py
+tridesclous/gui/spikelists.py
+tridesclous/gui/test_gpuselector.py
+tridesclous/gui/tools.py
+tridesclous/gui/traceviewer.py
+tridesclous/gui/waveformhistviewer.py
+tridesclous/gui/waveformviewer.py
+tridesclous/gui/icons/__init__.py
+tridesclous/gui/icons/icons_py3.py
+tridesclous/gui/icons/make_ressources.py
+tridesclous/gui/icons/test_ressources.py
+tridesclous/online/__init__.py
+tridesclous/online/launcher.py
+tridesclous/online/onlinepeeler.py
+tridesclous/online/onlinetools.py
+tridesclous/online/onlinetraceviewer.py
+tridesclous/online/onlinewaveformhistviewer.py
+tridesclous/online/onlinewindow.py
+tridesclous/online/probeactivityviewer.py
+tridesclous/scripts/__init__.py
+tridesclous/scripts/tdc.py
+tridesclous/tests/__init__.py
+tridesclous/tests/probe_list.py
+tridesclous/tests/test_autoparams.py
+tridesclous/tests/test_catalogueconstructor.py
+tridesclous/tests/test_cataloguetools.py
+tridesclous/tests/test_cleancluster.py
+tridesclous/tests/test_cltools.py
+tridesclous/tests/test_cluster.py
+tridesclous/tests/test_dataio.py
+tridesclous/tests/test_datasets.py
+tridesclous/tests/test_datasource.py
+tridesclous/tests/test_decomposition.py
+tridesclous/tests/test_dip.py
+tridesclous/tests/test_export.py
+tridesclous/tests/test_importers.py
+tridesclous/tests/test_iotools.py
+tridesclous/tests/test_jobtools.py
+tridesclous/tests/test_matplotlibplot.py
+tridesclous/tests/test_metrics.py
+tridesclous/tests/test_peakdetector.py
+tridesclous/tests/test_peeler.py
+tridesclous/tests/test_report.py
+tridesclous/tests/test_signalpreprocessor.py
+tridesclous/tests/test_tools.py
+tridesclous/tests/test_waveformtools.py
+tridesclous/tests/testingtools.py

linxi_tridesclous-1.0.0/linxi_tridesclous.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

linxi_tridesclous-1.0.0/linxi_tridesclous.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tdc = tridesclous.scripts.tdc:main

linxi_tridesclous-1.0.0/linxi_tridesclous.egg-info/requires.txt

@@ -0,0 +1,20 @@
+numpy
+scipy
+pandas
+openpyxl
+scikit-learn
+matplotlib
+seaborn
+neo
+tqdm
+pyqtgraph
+joblib
+numba
+hdbscan
+loky
+
+[online]
+pyacq
+
+[opencl]
+pyopencl

linxi_tridesclous-1.0.0/linxi_tridesclous.egg-info/top_level.txt

@@ -0,0 +1 @@
+tridesclous

linxi_tridesclous-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

linxi_tridesclous-1.0.0/setup.py

@@ -0,0 +1,44 @@
+from setuptools import setup
+import os
+
+d = {}
+exec(open("tridesclous/version.py").read(), None, d)
+version = d['version']
+
+install_requires = [
+                    'numpy',
+                    'scipy',
+                    'pandas',
+                    'openpyxl',
+                    'scikit-learn',
+                    'matplotlib',
+                    'seaborn',
+                    'neo',
+                    'tqdm',
+                    # 'PyQt5',  make conda buggy
+                    'pyqtgraph',
+                    'joblib',
+                    'numba',
+                    'hdbscan',
+                    'loky',
+                    ]
+extras_require={
+                            'online' : ['pyacq',],
+                            'opencl' : ['pyopencl'],
+                        }
+
+setup(
+    name = "linxi-tridesclous",
+    version = version,
+    packages = ['tridesclous', 'tridesclous.gui', 'tridesclous.gui.icons',
+                'tridesclous.online', 'tridesclous.scripts', 'tridesclous.tests'],
+    install_requires=install_requires,
+    extras_require = extras_require,
+    description = "offline/online spike sorting with french touch that light the barbecue",
+    entry_points={
+          'console_scripts': ['tdc=tridesclous.scripts.tdc:main'],
+          #~ 'gui_scripts': ['tdcgui=tridesclous.scripts.tdc:open_mainwindow'],
+        },
+    license = "MIT",
+    license_files = 'LICENSE',
+)

linxi_tridesclous-1.0.0/tridesclous/__init__.py

@@ -0,0 +1,43 @@
+"""
+
+.. automodule:: tridesclous.dataio
+
+.. automodule:: tridesclous.catalogueconstructor
+
+.. automodule:: tridesclous.peeler
+
+
+
+
+"""
+from .version import version as __version__
+import os
+import warnings
+
+from .datasets import download_dataset, get_dataset
+
+#dynamic import 
+from .datasource import data_source_classes
+for c in data_source_classes.values():
+    globals()[c.__name__] = c
+
+from .tools import open_prb
+from .dataio import DataIO
+from .signalpreprocessor import offline_signal_preprocessor, estimate_medians_mads_after_preprocesing
+from .catalogueconstructor import CatalogueConstructor
+from .cataloguetools import apply_all_catalogue_steps
+from .peeler import Peeler
+from .cltools import get_cl_device_list, set_default_cl_device
+
+from .autoparams import get_auto_params_for_catalogue, get_auto_params_for_peelers
+
+
+from .importers import import_from_spykingcircus, import_from_spike_interface
+
+# exclude because import matplotlib
+
+# from .matplotlibplot import *
+# from .report import *
+
+# from .gui import *
+

linxi_tridesclous-1.0.0/tridesclous/autoparams.py

@@ -0,0 +1,274 @@
+import time
+import copy
+from pprint import pprint
+
+import numpy as np
+
+from .cltools import HAVE_PYOPENCL
+from .cluster import HAVE_ISOSPLIT5
+
+try:
+    import numba
+    HAVE_NUMBA = True
+except ImportError:
+    HAVE_NUMBA = False
+
+
+# nb of channel to become sparse
+limit_dense_sparse = 4
+
+
+_default_catalogue_params = {
+    'duration': 300.,
+    
+    'chunksize': 1024,
+    'mode': 'dense', # 'sparse'
+    'sparse_threshold': None, # 1.5
+    'memory_mode': 'memmap',
+    'n_spike_for_centroid':350,
+    'n_jobs' :-1,
+
+    'preprocessor': {
+        'highpass_freq': 300.,
+        'lowpass_freq': 5000.,
+        'smooth_size': 0,
+        
+        'pad_width': -1,   # this auto
+        'engine': 'numpy',
+        'common_ref_removal':False,
+    },
+    'peak_detector': {
+        'method' : 'global',
+        'engine': 'numpy',
+        'peak_sign': '-',
+        'relative_threshold': 5.,
+        'peak_span_ms': .7,
+        'adjacency_radius_um' : None,
+        'smooth_radius_um' : None,
+        
+    },
+    'noise_snippet': {
+        'nb_snippet': 300,
+    },
+    'extract_waveforms': {
+        'wf_left_ms': -1.,
+        'wf_right_ms': 1.5,
+        'wf_left_long_ms': -2.5,
+        'wf_right_long_ms': 3.5,
+    },
+    'clean_peaks': {
+        #~ 'alien_value_threshold': None,
+        'alien_value_threshold':-1., # equivalent to None (normally)
+        #~ 'alien_value_threshold': np.nan,
+        'mode': 'extremum_amplitude',
+    },
+    'peak_sampler': {
+        'mode': 'rand',
+        'nb_max': 20000,
+        #~ 'nb_max_by_channel': None,
+        'nb_max_by_channel': 600.,
+    },
+    #~ 'clean_waveforms': {
+        #~ 'alien_value_threshold': None,
+    #~ },
+    'feature_method': 'pca_by_channel',
+    'feature_kargs': {},
+    'cluster_method': 'pruningshears',
+    'cluster_kargs': {},
+    
+    
+    'clean_cluster' :{
+        'apply_auto_split': True,
+        'apply_trash_not_aligned': True,
+        'apply_auto_merge_cluster': True,
+        'apply_trash_low_extremum': True,
+        'apply_trash_small_cluster': True,
+    },
+    
+    
+    'make_catalogue':{
+        'inter_sample_oversampling':False,
+        'subsample_ratio': 'auto',
+        'sparse_thresh_level2': 1.5,
+        'sparse_thresh_level2': 3,
+    }
+}
+
+
+
+
+
+def get_auto_params_for_catalogue(dataio=None, chan_grp=0,
+                            nb_chan=None, sample_rate=None,
+                            context='offline'):
+    """
+    Automatic selection of parameters.
+    This totally empiric paramerters.
+    """
+    
+    assert context in ('offline', 'online')
+    
+    params = copy.deepcopy(_default_catalogue_params)
+
+    # TODO make this more complicated
+    #  * by detecting if dense array or not.
+    #  * better method sleection
+
+
+    if dataio is None:
+        # case online no DataIO
+        assert nb_chan is not None
+        assert sample_rate is not None
+        # use less because chunksize is set outsied
+        params['chunksize'] = int(sample_rate * 0.05)
+        params['duration'] = 10. # debug
+    else:
+        # case offline with a DataIO
+        nb_chan = dataio.nb_channel(chan_grp=chan_grp)
+        sample_rate = dataio.sample_rate
+        total_duration = sum(dataio.get_segment_length(seg_num=seg_num) / dataio.sample_rate for seg_num in range(dataio.nb_segment))
+        # auto chunsize of 100 ms
+        params['chunksize'] = int(sample_rate * 1.0)
+        params['duration'] = 601.
+        
+        # segment durartion is not so big then take the whole duration
+        # to avoid double preprocessing (catalogue+peeler)
+        if params['duration'] * 2 > total_duration:
+            params['duration'] = total_duration
+    
+    #~ if nb_chan == 1:
+        #~ params['mode'] = 'dense'
+        #~ params['adjacency_radius_um'] = 0.
+        #~ params['sparse_threshold'] = 1.5
+        
+        #~ params['peak_detector']['method'] = 'global'
+        #~ params['peak_detector']['engine'] = 'numpy'
+        #~ params['peak_detector']['smooth_radius_um' ] = None
+
+
+        #~ params['peak_sampler']['mode'] = 'rand'
+        #~ params['peak_sampler']['nb_max'] = 20000
+        
+        #~ params['feature_method'] = 'global_pca'
+        #~ params['feature_kargs'] = {'n_components' : 4 }
+        
+        #~ params['cluster_method'] = 'dbscan_with_noise'
+        #~ params['cluster_kargs'] = {}
+        
+
+        
+        #~ params['clean_cluster_kargs'] = {'too_small' : 20 }        
+    
+    #~ elif nb_chan <=4:
+    if nb_chan <= limit_dense_sparse:
+    #~ if nb_chan <=8:
+    
+        params['mode'] = 'dense'
+        #~ params['adjacency_radius_um'] = 0.
+        params['sparse_threshold'] = 1.5
+        
+        #~ params['peak_detector']['method'] = 'global'
+        #~ params['peak_detector']['engine'] = 'numpy'
+
+        params['peak_detector']['method'] = 'geometrical'
+        params['peak_detector']['engine'] = 'numba'
+        
+        params['peak_detector']['adjacency_radius_um'] = 200. # useless
+        params['peak_detector']['smooth_radius_um' ] = None
+
+
+        params['peak_sampler']['mode'] = 'rand'
+        params['peak_sampler']['nb_max'] = 20000
+        
+        params['feature_method'] = 'global_pca'
+        if nb_chan in (1,2):
+            n_components = 5
+        else:
+            n_components = int(nb_chan*2)
+        params['feature_kargs'] = {'n_components' : n_components }
+        
+        
+        params['cluster_method'] = 'pruningshears'
+        params['cluster_kargs']['max_loop'] = max(1000, nb_chan * 10)
+        params['cluster_kargs']['min_cluster_size'] = 20
+        params['cluster_kargs']['adjacency_radius_um'] = 0.
+        params['cluster_kargs']['high_adjacency_radius_um'] = 0.
+        
+        # necessary for peeler classic
+        #~ params['make_catalogue']['inter_sample_oversampling'] = True
+
+
+        
+
+    else:
+        params['mode'] = 'sparse'
+        #~ params['adjacency_radius_um'] = 200.
+        params['sparse_threshold'] = 1.5
+
+        if nb_chan >= 32 and HAVE_PYOPENCL:
+            params['preprocessor']['engine'] = 'opencl'
+        params['peak_detector']['method'] = 'geometrical'
+        params['peak_detector']['adjacency_radius_um'] = 200.
+        #~ params['peak_detector']['smooth_radius_um' ] = 10
+        params['peak_detector']['smooth_radius_um' ] = None
+        
+        
+        if HAVE_PYOPENCL:
+            params['peak_detector']['engine'] = 'opencl'
+        elif HAVE_NUMBA:
+            params['peak_detector']['engine'] = 'numba'
+        else:
+            print('WARNING : peakdetector will be slow install opencl or numba')
+            params['peak_detector']['engine'] = 'numpy'
+        
+        params['peak_sampler']['mode'] = 'rand_by_channel'
+        #~ params['extract_waveforms']['nb_max_by_channel'] = 700
+        params['peak_sampler']['nb_max_by_channel'] = 1000
+        #~ params['peak_sampler']['nb_max_by_channel'] = 1500
+        #~ params['peak_sampler']['nb_max_by_channel'] = 3000
+        
+        
+        params['feature_method'] = 'pca_by_channel'
+        # TODO change n_components_by_channel depending on channel density
+        #~ params['feature_kargs'] = {'n_components_by_channel':5}
+        params['feature_kargs'] = {'n_components_by_channel': 3,
+                                                        'adjacency_radius_um' :50.,  # this should be greater than cluster 'adjacency_radius_um'
+                                                        }
+        
+        params['cluster_method'] = 'pruningshears'
+        params['cluster_kargs']['max_loop'] = max(1000, nb_chan * 20)
+        params['cluster_kargs']['min_cluster_size'] = 20
+        params['cluster_kargs']['adjacency_radius_um'] = 50.
+        params['cluster_kargs']['high_adjacency_radius_um'] = 30.
+
+    if context == 'online':
+        params['n_jobs' ] = 1
+    
+    return params
+
+
+
+def get_auto_params_for_peelers(dataio, chan_grp=0):
+    nb_chan = dataio.nb_channel(chan_grp=chan_grp)
+    params = {}
+    
+    params['chunksize'] = int(dataio.sample_rate * 1)
+    #~ params['chunksize'] = int(dataio.sample_rate * 0.5)
+    #~ params['chunksize'] = int(dataio.sample_rate * 0.1)
+    #~ params['chunksize'] = int(dataio.sample_rate * 0.033)
+
+    if nb_chan <= limit_dense_sparse:
+        params['engine'] = 'geometrical'
+        
+    else:
+        if HAVE_PYOPENCL:
+            params['engine'] = 'geometrical_opencl'
+        else:
+            params['engine'] = 'geometrical'
+
+    
+    # DEBUG force 'geometrical'
+    params['engine'] = 'geometrical'
+        
+
+    return params