ibl-neuropixel 1.9.3__py3-none-any.whl → 1.10.0__py3-none-any.whl

tests/unit/test_ephys_np2.py

@@ -1,29 +1,63 @@
 import random
 import shutil
 import unittest
+import tempfile
 from pathlib import Path
 
 import numpy as np
 from neuropixel import NP2Converter, NP2Reconstructor
 import spikeglx
 
+FIXTURE_PATH = Path(__file__).parents[1].joinpath("fixtures", "np2split")
+
 
 class BaseEphysNP2(unittest.TestCase):
-    data_path = Path(__file__).parents[1].joinpath("fixtures", "np2split")
+    nc = None
+    folder_test_case = None
 
     @classmethod
     def setUpClass(cls):
-        dat = np.tile(np.arange(385)[np.newaxis, :] + 10000, [30000, 1]).astype(
+        # create a temporary directory for the test
+        cls._temp_dir_obj = tempfile.TemporaryDirectory(prefix="test_np2_")
+        cls.data_path = Path(cls._temp_dir_obj.name)
+        # we create a small dummy spikeglx file for testing
+        cls.orig_file = cls.data_path.joinpath("_spikeglx_ephysData_g0_t0.imec0.ap.bin")
+        # the metadata file is copied from fixtures
+        cls.orig_meta_file = cls.orig_file.with_suffix(".meta")
+        shutil.copy(
+            FIXTURE_PATH.joinpath(
+                cls.folder_test_case, "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
+            ),
+            cls.orig_meta_file,
+        )
+        # generation of a dummy binary file for testing
+        dat = np.tile(np.arange(cls.nc)[np.newaxis, :] + 10000, [30000, 1]).astype(
             np.int16
         )
-        with open(
-            cls.data_path.joinpath("_spikeglx_ephysData_g0_t0.imec0.ap.bin"), "bw+"
-        ) as fid:
+        with open(cls.orig_file, "bw+") as fid:
             dat.tofile(fid)
 
     @classmethod
     def tearDownClass(cls):
-        cls.data_path.joinpath("_spikeglx_ephysData_g0_t0.imec0.ap.bin").unlink()
+        cls._temp_dir_obj.cleanup()
+
+    def setUp(self):
+        """
+        :param folder_test_case: NP1_meta | NP21_meta | NP24_meta
+        :return:
+        """
+        self._temp_dir_obj_case = tempfile.TemporaryDirectory(prefix="test_np2_case_")
+        current_case_path = Path(self._temp_dir_obj_case.name).joinpath("probe00")
+        current_case_path.mkdir(parents=True, exist_ok=True)
+        self.file_path = current_case_path.joinpath(self.orig_file.name)
+        self.meta_file = self.file_path.with_suffix(".meta")
+        shutil.copy(self.orig_file, self.file_path)
+        shutil.copy(self.orig_meta_file, self.meta_file)
+        self.sglx_instances = []
+
+    def tearDown(self):
+        self._temp_dir_obj_case.cleanup()
+        _ = [sglx.close() for sglx in self.sglx_instances]
 
 
 class TestNeuropixel2ConverterNP24(BaseEphysNP2):
@@ -31,30 +65,8 @@ class TestNeuropixel2ConverterNP24(BaseEphysNP2):
     Check NP2 converter with NP2.4 type probes
     """
 
-    def setUp(self) -> None:
-        file_path = self.data_path.joinpath("_spikeglx_ephysData_g0_t0.imec0.ap.bin")
-        self.file_path = file_path.parent.parent.joinpath(
-            "probe00_temp", file_path.name
-        )
-        self.file_path.parent.mkdir(exist_ok=True, parents=True)
-        meta_file = file_path.parent.joinpath(
-            "NP24_meta", "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        self.meta_file = self.file_path.parent.joinpath(
-            "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        shutil.copy(file_path, self.file_path)
-        shutil.copy(meta_file, self.meta_file)
-        self.sglx_instances = []
-
-    def tearDown(self):
-        _ = [sglx.close() for sglx in self.sglx_instances]
-        # here should look for any directories with test in it and delete
-        test_dir = list(self.file_path.parent.parent.glob("*test*"))
-        _ = [shutil.rmtree(test) for test in test_dir]
-        # For case where we have deleted already as part of test
-        if self.file_path.parent.exists():
-            shutil.rmtree(self.file_path.parent)
+    nc = 385
+    folder_test_case = "NP24_meta"
 
     def testDecimate(self):
         """
@@ -62,7 +74,6 @@ class TestNeuropixel2ConverterNP24(BaseEphysNP2):
         window lengths for iterating through data
         :return:
         """
-
         FS = 30000
         np_a = NP2Converter(self.file_path, post_check=False, compress=False)
         np_a.init_params(nwindow=0.3 * FS, extra="_0_5s_test", nshank=[0])
@@ -189,7 +200,6 @@ class TestNeuropixel2ConverterNP24(BaseEphysNP2):
         Check that if the data has been incorrectly split we get a warning error
         :return:
         """
-
         np_conv = NP2Converter(self.file_path, compress=False)
         np_conv.init_params(extra="_test")
         status = np_conv.process()
@@ -204,10 +214,7 @@ class TestNeuropixel2ConverterNP24(BaseEphysNP2):
         # Now that we have changed the file we expect an assertion error when we do the check
         with self.assertRaises(AssertionError) as context:
             np_conv.check_NP24()
-        self.assertTrue(
-            "data in original file and split files do no match"
-            in str(context.exception)
-        )
+        self.assertTrue("Arrays are not equal" in str(context.exception))
 
     def testFromCompressed(self):
         """
@@ -232,26 +239,8 @@ class TestNeuropixel2ConverterNP21(BaseEphysNP2):
     Check NP2 converter with NP2.1 type probes
     """
 
-    def setUp(self) -> None:
-        file_path = self.data_path.joinpath("_spikeglx_ephysData_g0_t0.imec0.ap.bin")
-        self.file_path = file_path.parent.parent.joinpath(
-            "probe00_temp", file_path.name
-        )
-        self.file_path.parent.mkdir(exist_ok=True, parents=True)
-        meta_file = file_path.parent.joinpath(
-            "NP21_meta", "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        self.meta_file = self.file_path.parent.joinpath(
-            "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        shutil.copy(file_path, self.file_path)
-        shutil.copy(meta_file, self.meta_file)
-        self.sglx_instances = []
-
-    def tearDown(self):
-        _ = [sglx.close() for sglx in self.sglx_instances]
-        # here should look for anything with test in it and delete
-        shutil.rmtree(self.file_path.parent)
+    nc = 385
+    folder_test_case = "NP21_meta"
 
     def testProcessNP21(self):
         """
@@ -293,29 +282,13 @@ class TestNeuropixel2ConverterNP21(BaseEphysNP2):
         np_conv.sr.close()
 
 
-class TestNeuropixel2ConverterNP1(NP2Converter):
+class TestNeuropixel2ConverterNP1(BaseEphysNP2):
     """
     Check NP2 converter with NP1 type probes
     """
 
-    def setUp(self) -> None:
-        self.file_path = self.data_path.joinpath(
-            "_spikeglx_ephysData_g0_t0.imec0.ap.bin"
-        )
-        meta_file = self.file_path.parent.joinpath(
-            "NP1_meta", "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        self.meta_file = self.file_path.with_suffix(".meta")
-        # Back up current meta file
-        shutil.move(self.meta_file, self.meta_file.with_suffix(".meta.bk"))
-        # Copy the neuropixels v1 meta file into the probe00 folder
-        shutil.copy(meta_file, self.meta_file)
-        self.sglx_instances = []
-        self.temp_directories = []
-
-    def tearDown(self):
-        # replace meta file with backup
-        shutil.move(self.meta_file.with_suffix(".meta.bk"), self.meta_file)
+    nc = 385
+    folder_test_case = "NP1_meta"
 
     def testProcessNP1(self):
         """
@@ -327,38 +300,15 @@ class TestNeuropixel2ConverterNP1(NP2Converter):
 
 
 class TestNeuropixelReconstructor(BaseEphysNP2):
-    def setUp(self) -> None:
-        self.orig_file = self.data_path.joinpath(
-            "_spikeglx_ephysData_g0_t0.imec0.ap.bin"
-        )
-        self.file_path = self.orig_file.parent.parent.joinpath(
-            "probe00_temp", self.orig_file.name
-        )
-        self.file_path.parent.mkdir(exist_ok=True, parents=True)
-        self.orig_meta_file = self.orig_file.parent.joinpath(
-            "NP24_meta", "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        self.meta_file = self.file_path.parent.joinpath(
-            "_spikeglx_ephysData_g0_t0.imec0.ap.meta"
-        )
-        shutil.copy(self.orig_file, self.file_path)
-        shutil.copy(self.orig_meta_file, self.meta_file)
-        self.sglx_instances = []
-
-    def tearDown(self):
-        _ = [sglx.close() for sglx in self.sglx_instances]
-        # here should look for any directories with test in it and delete
-        test_dir = list(self.file_path.parent.parent.glob("*test*"))
-        _ = [shutil.rmtree(test) for test in test_dir]
-        # For case where we have deleted already as part of test
-        if self.file_path.parent.exists():
-            shutil.rmtree(self.file_path.parent)
+    nc = 385
+    folder_test_case = "NP24_meta"
 
     def test_reconstruction(self):
         # First split the probes
         np_conv = NP2Converter(self.file_path)
         np_conv.init_params(extra="_test")
-        _ = np_conv.process()
+        status = np_conv.process()
+        self.assertTrue(status)
         np_conv.sr.close()
 
         # Delete the original file
@@ -367,7 +317,7 @@ class TestNeuropixelReconstructor(BaseEphysNP2):
 
         # Now reconstruct
         np_recon = NP2Reconstructor(
-            self.file_path.parent.parent, pname="probe00_temp", compress=True
+            self.file_path.parents[1], pname="probe00", compress=True
         )
         status = np_recon.process()
 
@@ -388,7 +338,27 @@ class TestNeuropixelReconstructor(BaseEphysNP2):
         self.assertEqual(orig_meta, recon_meta)
 
 
-if __name__ == "__main__":
-    import unittest
+class TestNeuropixel2ConverterNP2QB(BaseEphysNP2):
+    nc = 385 * 4
+    folder_test_case = "NP2QB_meta"
+
+    def testProcessNP2QB(self):
+        # Make sure normal workflow runs without problems
+        np_conv = NP2Converter(self.file_path)
+        np_conv.init_params(extra="_test")
+        status = np_conv.process()
+        self.assertFalse(np_conv.already_exists)
+        self.assertTrue(status)
 
-    unittest.main(exit=False)
+        # Test a random ap metadata file and make sure it all makes sense
+        shank_n = random.randint(0, 3)
+        sr_ap = spikeglx.Reader(
+            np_conv.shank_info[f"shank{shank_n}"]["ap_file"], sort=False
+        )
+        np.testing.assert_array_equal(sr_ap.meta["acqApLfSy"], [384, 0, 1])
+        np.testing.assert_array_equal(sr_ap.meta["snsApLfSy"], [384, 0, 1])
+        self.assertEqual(sr_ap.meta["nSavedChans"], 385)
+        self.assertEqual(sr_ap.meta["snsSaveChanSubset"], "0:384")
+        self.assertEqual(sr_ap.meta["NP2.4_shank"], shank_n)
+        self.assertEqual(sr_ap.meta["original_meta"], "False")
+        sr_ap.close()

tests/unit/test_neuropixel.py

@@ -1,5 +1,6 @@
 import neuropixel
 import numpy as np
+import re
 
 
 def test_sites_coordinates_deprecated():
@@ -82,3 +83,39 @@ def test_geom_npultra():
     h = neuropixel.trace_header("NPultra")
     for k, v in gt.items():
         np.testing.assert_equal(v, h[k])
+
+
+def test_geom_quad():
+    assert "NP2QB" in neuropixel.CHANNEL_GRID.keys()
+    neuropixel.adc_shifts(version="NP2QB")
+    pass
+
+
+def test_get_probe_table():
+    df_tables, probe_table = neuropixel.load_spike_glx_probe_table()
+    df_tables = df_tables.loc[df_tables["is_commercial"] == "Y", :]
+    df_tables["mux_table_format_type"].unique()
+
+    # Get the the mux tables from the probe tables
+    def get_mux_table(mux_string):
+        groups = re.findall(r"\(([0-9\s]+)\)", mux_string)
+        mux = np.array([[int(x) for x in g.split()] for g in groups], dtype=np.int32)
+        adc_vector = np.zeros(mux.size)
+        for i in range(mux.shape[1]):
+            adc_vector[mux[:, i]] = i
+        return adc_vector
+
+    # Check neuropixel 1
+    th1 = neuropixel.trace_header(version=1)
+    mux1 = get_mux_table(probe_table["z_mux_tables"]["mux_np1000"])
+    np.testing.assert_allclose(np.diff(np.c_[th1["adc"], mux1], axis=1), 0)
+
+    # Check Neuropixel 2
+    th2 = neuropixel.trace_header(version=2)
+    mux2 = get_mux_table(probe_table["z_mux_tables"]["mux_np2000"])
+    np.testing.assert_allclose(np.diff(np.c_[th2["adc"], mux2], axis=1), 0)
+
+    # Check Neuropixel 2 Quadbase
+    thqb = neuropixel.trace_header(version="NP2QB")
+    muxqb = get_mux_table(probe_table["z_mux_tables"]["mux_np2020"])
+    np.testing.assert_allclose(np.diff(np.c_[thqb["adc"], muxqb], axis=1), 0)

tests/unit/test_spikeglx.py

@@ -382,11 +382,11 @@ class TestsSpikeGLX_Meta(unittest.TestCase):
             )
             self.assert_read_glx(bin_3b)
 
-    def test_read_NP20_prototype(self):
+    def test_read_NHP_prototype(self):
         with tempfile.TemporaryDirectory(prefix="glx_test") as tdir:
             bin_3b = spikeglx._mock_spikeglx_file(
-                Path(tdir).joinpath("sampleNP2.1_prototype.ap.bin"),
-                self.workdir / "sampleNP2.1_prototype.ap.meta",
+                Path(tdir).joinpath("sampleNHPlong_prototype.ap.bin"),
+                self.workdir / "sampleNHPlong_prototype.ap.meta",
                 ns=32,
                 nc=385,
                 sync_depth=16,
@@ -516,7 +516,7 @@ class TestsSpikeGLX_Meta(unittest.TestCase):
                 )
                 for k in h_expected.keys():
                     np.testing.assert_equal(
-                        h_expected[k][sr.raw_channel_order[: -sr.nsync]], th[k]
+                        th[k], h_expected[k][sr.raw_channel_order[: -sr.nsync]]
                     )
 
     def testGetSerialNumber(self):
@@ -557,7 +557,16 @@ class TestsSpikeGLX_Meta(unittest.TestCase):
                             spikeglx._get_neuropixel_major_version_from_meta(md),
                         )
                         continue
-
+                    elif "NP2QB" in meta_data_file.name:
+                        self.assertEqual(
+                            "NP2QB", spikeglx._get_neuropixel_version_from_meta(md)
+                        )
+                        continue
+                    elif "NHPlong" in meta_data_file.name:
+                        self.assertEqual(
+                            "NHPlong", spikeglx._get_neuropixel_version_from_meta(md)
+                        )
+                        continue
                     # for ap and lf look for version number
                     # test getting revision
                     revision = meta_data_file.name[6:8]
@@ -580,12 +589,16 @@ class TestsSpikeGLX_Meta(unittest.TestCase):
                     continue
                 md = spikeglx.read_meta_data(meta_data_file)
                 cg = spikeglx._conversion_sample2v_from_meta(md)
+                if spikeglx._get_neuropixel_version_from_meta(md) == "NP2QB":
+                    continue
                 if "NP2" in spikeglx._get_neuropixel_version_from_meta(md):
+                    gain = spikeglx._get_gain_from_meta(md)
                     maxint = spikeglx._get_max_int_from_meta(md)
+                    print(meta_data_file.name, maxint, gain)
                     self.assertEqual(int(md.get("imMaxInt")), maxint)
                     i2v = md.get("imAiRangeMax") / maxint
-                    self.assertTrue(np.all(cg["lf"][0:-1] == i2v / 80))
-                    self.assertTrue(np.all(cg["ap"][0:-1] == i2v / 80))
+                    self.assertTrue(np.all(cg["lf"][0:-1] == i2v / gain))
+                    self.assertTrue(np.all(cg["ap"][0:-1] == i2v / gain))
                 else:
                     maxint = spikeglx._get_max_int_from_meta(md)
                     self.assertEqual(512, maxint)

tests/unit/test_sync.py

@@ -0,0 +1,104 @@
+import datetime
+import unittest
+import tempfile
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+import spikeglx
+import ibldsp.sync
+
+TEST_PATH = Path(__file__).parents[1].joinpath("fixtures")
+
+
+class TestSyncTimestamps(unittest.TestCase):
+    def test_deprecation(self):
+        if datetime.datetime.now() > datetime.datetime(2026, 10, 12):
+            raise NotImplementedError(
+                "Time to deprecate ibldsp.utils.sync_timestamps()"
+            )
+
+    def test_sync_timestamps_linear(self):
+        ta = np.cumsum(np.abs(np.random.randn(100))) * 10
+        tb = ta * 1.0001 + 100
+        fcn, drif, ia, ib = ibldsp.sync.sync_timestamps(
+            ta, tb, return_indices=True, linear=True
+        )
+        np.testing.assert_almost_equal(drif, 100)
+        np.testing.assert_almost_equal(tb, fcn(ta))
+
+    def test_timestamps(self):
+        np.random.seed(4132)
+        n = 50
+        drift = 17.14
+        offset = 34.323
+        tsa = np.cumsum(np.random.random(n) * 10)
+        tsb = tsa * (1 + drift / 1e6) + offset
+
+        # test linear drift
+        _fcn, _drift = ibldsp.sync.sync_timestamps(tsa, tsb)
+        assert np.all(np.isclose(_fcn(tsa), tsb))
+        assert np.isclose(drift, _drift)
+
+        # test missing indices on a
+        imiss = np.setxor1d(np.arange(n), [1, 2, 34, 35])
+        _fcn, _drift, _ia, _ib = ibldsp.sync.sync_timestamps(
+            tsa[imiss], tsb, return_indices=True
+        )
+        assert np.all(np.isclose(_fcn(tsa[imiss[_ia]]), tsb[_ib]))
+
+        # test missing indices on b
+        _fcn, _drift, _ia, _ib = ibldsp.sync.sync_timestamps(
+            tsa, tsb[imiss], return_indices=True
+        )
+        assert np.all(np.isclose(_fcn(tsa[_ia]), tsb[imiss[_ib]]))
+
+        # test missing indices on both
+        imiss2 = np.setxor1d(np.arange(n), [14, 17])
+        _fcn, _drift, _ia, _ib = ibldsp.sync.sync_timestamps(
+            tsa[imiss], tsb[imiss2], return_indices=True
+        )
+        assert np.all(np.isclose(_fcn(tsa[imiss[_ia]]), tsb[imiss2[_ib]]))
+
+        # test timestamps with huge offset (previously caused ArrayMemoryError)
+        # tsb -= 1e15
+        # _fcn, _drift = utils.sync_timestamps(tsa, tsb)
+        # assert np.all(np.isclose(_fcn(tsa), tsb))
+
+
+class TestSyncSpikeGlx:
+    # def setUp(self):
+    #     self.workdir = Path(__file__).parents[1] / 'fixtures' / 'sync_ephys_fpga'
+    #     self.meta_files = list(Path.glob(self.workdir, '*.meta'))
+
+    def test_sync_nidq(self):
+        self.sync_gen(fn="sample3B_g0_t0.nidq.meta", ns=32, nc=2, sync_depth=8)
+
+    def test_sync_NP1(self):
+        self.sync_gen(fn="sample3B_g0_t0.imec1.ap.meta", ns=32, nc=385, sync_depth=16)
+
+    def sync_gen(self, fn, ns, nc, sync_depth):
+        # nidq has 1 analog and 1 digital sync channels
+        with tempfile.TemporaryDirectory() as tdir:
+            ses_path = Path(tdir).joinpath("raw_ephys_data")
+            ses_path.mkdir(parents=True, exist_ok=True)
+            meta_file = ses_path.joinpath(fn)
+            bin_file = meta_file.with_suffix(".bin")
+            import shutil
+
+            shutil.copy(TEST_PATH.joinpath(fn), meta_file)
+            _ = spikeglx._mock_spikeglx_file(
+                bin_file,
+                meta_file=TEST_PATH.joinpath(fn),
+                ns=ns,
+                nc=nc,
+                sync_depth=sync_depth,
+            )
+            sr = spikeglx.Reader(bin_file)
+            # for a nidq file, there can be additional analog sync channels shown in the sync
+            csel = spikeglx._get_analog_sync_trace_indices_from_meta(sr.meta)
+            df_sync = pd.DataFrame(ibldsp.sync.extract_spikeglx_sync(bin_file))
+            np.testing.assert_equal(
+                len(df_sync["channels"].unique()), sync_depth + len(csel)
+            )

tests/unit/test_utils.py

@@ -18,55 +18,6 @@ import shutil
 FIXTURE_PATH = Path(__file__).parents[1].joinpath("fixtures")
 
 
-class TestSyncTimestamps(unittest.TestCase):
-    def test_sync_timestamps_linear(self):
-        ta = np.cumsum(np.abs(np.random.randn(100))) * 10
-        tb = ta * 1.0001 + 100
-        fcn, drif, ia, ib = utils.sync_timestamps(
-            ta, tb, return_indices=True, linear=True
-        )
-        np.testing.assert_almost_equal(drif, 100)
-        np.testing.assert_almost_equal(tb, fcn(ta))
-
-    def test_timestamps(self):
-        np.random.seed(4132)
-        n = 50
-        drift = 17.14
-        offset = 34.323
-        tsa = np.cumsum(np.random.random(n) * 10)
-        tsb = tsa * (1 + drift / 1e6) + offset
-
-        # test linear drift
-        _fcn, _drift = utils.sync_timestamps(tsa, tsb)
-        assert np.all(np.isclose(_fcn(tsa), tsb))
-        assert np.isclose(drift, _drift)
-
-        # test missing indices on a
-        imiss = np.setxor1d(np.arange(n), [1, 2, 34, 35])
-        _fcn, _drift, _ia, _ib = utils.sync_timestamps(
-            tsa[imiss], tsb, return_indices=True
-        )
-        assert np.all(np.isclose(_fcn(tsa[imiss[_ia]]), tsb[_ib]))
-
-        # test missing indices on b
-        _fcn, _drift, _ia, _ib = utils.sync_timestamps(
-            tsa, tsb[imiss], return_indices=True
-        )
-        assert np.all(np.isclose(_fcn(tsa[_ia]), tsb[imiss[_ib]]))
-
-        # test missing indices on both
-        imiss2 = np.setxor1d(np.arange(n), [14, 17])
-        _fcn, _drift, _ia, _ib = utils.sync_timestamps(
-            tsa[imiss], tsb[imiss2], return_indices=True
-        )
-        assert np.all(np.isclose(_fcn(tsa[imiss[_ia]]), tsb[imiss2[_ib]]))
-
-        # test timestamps with huge offset (previously caused ArrayMemoryError)
-        # tsb -= 1e15
-        # _fcn, _drift = utils.sync_timestamps(tsa, tsb)
-        # assert np.all(np.isclose(_fcn(tsa), tsb))
-
-
 class TestParabolicMax(unittest.TestCase):
     # expected values
     maxi = np.array([0.0, 0.0, 3.04166667, 3.04166667, 5, 5])