datapack-neuro 1.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. datapack_neuro-1.2.0/.gitignore +71 -0
  2. datapack_neuro-1.2.0/CITATION.cff +37 -0
  3. datapack_neuro-1.2.0/LICENSE +32 -0
  4. datapack_neuro-1.2.0/PKG-INFO +226 -0
  5. datapack_neuro-1.2.0/README.md +163 -0
  6. datapack_neuro-1.2.0/VERSION +1 -0
  7. datapack_neuro-1.2.0/pyproject.toml +154 -0
  8. datapack_neuro-1.2.0/src/datapack/__init__.py +124 -0
  9. datapack_neuro-1.2.0/src/datapack/_version.py +24 -0
  10. datapack_neuro-1.2.0/src/datapack/cli.py +379 -0
  11. datapack_neuro-1.2.0/src/datapack/config.py +62 -0
  12. datapack_neuro-1.2.0/src/datapack/core/__init__.py +41 -0
  13. datapack_neuro-1.2.0/src/datapack/core/atomic.py +90 -0
  14. datapack_neuro-1.2.0/src/datapack/core/detect.py +55 -0
  15. datapack_neuro-1.2.0/src/datapack/core/engine.py +39 -0
  16. datapack_neuro-1.2.0/src/datapack/core/errors.py +40 -0
  17. datapack_neuro-1.2.0/src/datapack/core/job.py +37 -0
  18. datapack_neuro-1.2.0/src/datapack/core/manifest.py +83 -0
  19. datapack_neuro-1.2.0/src/datapack/core/verify.py +45 -0
  20. datapack_neuro-1.2.0/src/datapack/datasets.py +143 -0
  21. datapack_neuro-1.2.0/src/datapack/engines/__init__.py +53 -0
  22. datapack_neuro-1.2.0/src/datapack/engines/ephys.py +595 -0
  23. datapack_neuro-1.2.0/src/datapack/engines/histology.py +584 -0
  24. datapack_neuro-1.2.0/src/datapack/engines/video.py +433 -0
  25. datapack_neuro-1.2.0/src/datapack/py.typed +0 -0
  26. datapack_neuro-1.2.0/tests/conftest.py +1 -0
  27. datapack_neuro-1.2.0/tests/integration/test_ephys_roundtrip.py +92 -0
  28. datapack_neuro-1.2.0/tests/integration/test_histology.py +210 -0
  29. datapack_neuro-1.2.0/tests/integration/test_oe_session.py +227 -0
  30. datapack_neuro-1.2.0/tests/integration/test_placeholder.py +6 -0
  31. datapack_neuro-1.2.0/tests/integration/test_serial_section.py +116 -0
  32. datapack_neuro-1.2.0/tests/integration/test_video_encode.py +61 -0
  33. datapack_neuro-1.2.0/tests/unit/test_cli.py +53 -0
  34. datapack_neuro-1.2.0/tests/unit/test_core.py +219 -0
  35. datapack_neuro-1.2.0/tests/unit/test_ephys.py +78 -0
  36. datapack_neuro-1.2.0/tests/unit/test_video.py +117 -0
@@ -0,0 +1,71 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.pyo
5
+ *.pyd
6
+ *.so
7
+ *.egg
8
+ *.egg-info/
9
+ dist/
10
+ build/
11
+ eggs/
12
+ parts/
13
+ var/
14
+ sdist/
15
+ develop-eggs/
16
+ .installed.cfg
17
+ lib/
18
+ lib64/
19
+ wheels/
20
+
21
+ # Virtual environments
22
+ .venv/
23
+ venv/
24
+ env/
25
+ ENV/
26
+ .python-version
27
+
28
+ # uv
29
+ uv.lock
30
+
31
+ # Distribution / packaging
32
+ MANIFEST
33
+
34
+ # Testing
35
+ .pytest_cache/
36
+ .coverage
37
+ .coverage.*
38
+ coverage.xml
39
+ htmlcov/
40
+ *.cover
41
+ .hypothesis/
42
+
43
+ # Type checking
44
+ .mypy_cache/
45
+ .dmypy.json
46
+ dmypy.json
47
+ .pytype/
48
+ .pyre/
49
+
50
+ # Build artifacts (hatch-vcs generated)
51
+ src/*/_version.py
52
+
53
+ # Jupyter
54
+ .ipynb_checkpoints
55
+ *.ipynb
56
+
57
+ # IDEs
58
+ .idea/
59
+ .vscode/
60
+ *.swp
61
+ *.swo
62
+ *~
63
+
64
+ # OS
65
+ .DS_Store
66
+ Thumbs.db
67
+
68
+ # Secrets / local config
69
+ .env
70
+ .env.*
71
+ !.env.example
@@ -0,0 +1,37 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use this software, please cite it as below."
3
+ type: software
4
+ title: "datapack"
5
+ version: "1.1.0"
6
+ repository-code: https://github.com/larsrollik/datapack
7
+ url: https://github.com/larsrollik/datapack
8
+ # Not an SPDX identifier: the terms are custom (non-commercial research use only),
9
+ # so CFF takes a URL rather than an id. Declaring GPL-3.0 here, as this file used to,
10
+ # contradicted LICENSE and would have told a reader they could use this commercially.
11
+ license-url: https://github.com/larsrollik/datapack/blob/main/LICENSE
12
+ authors:
13
+ - family-names: Rollik
14
+ given-names: Lars
15
+ # orcid: add before the first tagged release; a placeholder id must not ship
16
+
17
+ # Zenodo integration (optional):
18
+ # 1. Enable the Zenodo webhook at zenodo.org/account/settings/github
19
+ # 2. After first tag release, copy the concept DOI from Zenodo and add:
20
+ #
21
+ # identifiers:
22
+ # - type: doi
23
+ # value: 10.5281/zenodo.XXXXXXX
24
+ # description: Zenodo concept DOI
25
+ #
26
+ # preferred-citation:
27
+ # type: software
28
+ # title: "datapack"
29
+ # authors:
30
+ # - family-names: ...
31
+ # given-names: ...
32
+ # orcid: https://orcid.org/...
33
+ # year: 2026
34
+ # publisher:
35
+ # name: Zenodo
36
+ # doi: 10.5281/zenodo.XXXXXXX
37
+ # url: https://doi.org/10.5281/zenodo.XXXXXXX
@@ -0,0 +1,32 @@
1
+
2
+ Copyright (c) 2026-present Lars B. Rollik. All rights reserved.
3
+
4
+ Permission is granted, free of charge, to use, copy, modify, and distribute
5
+ this software and associated documentation files (the "Software") for
6
+ non-commercial research or academic purposes only, subject to the following
7
+ conditions:
8
+
9
+ 1. This copyright notice and permission notice must be included in all copies
10
+ or substantial portions of the Software.
11
+
12
+ 2. Any publication, presentation, or product that uses or builds upon the
13
+ Software must give clear attribution to the original work and its authors.
14
+
15
+ 3. Commercial use is prohibited without a separate written commercial licence
16
+ agreement with the copyright holder. Commercial use means incorporation of
17
+ the Software into anything for which fees or other compensation are charged
18
+ or received, including commercial products and commercial services.
19
+
20
+ 4. No patent licence, express or implied, is granted under these terms. Any
21
+ use that would require a patent licence from the copyright holder requires
22
+ a separate written agreement.
23
+
24
+ 5. Redistribution, in source or binary form, is permitted only for
25
+ non-commercial purposes and must retain this notice unmodified.
26
+
27
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28
+ IMPLIED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
29
+ DAMAGES, OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THE
30
+ SOFTWARE OR THE USE OR DEALINGS IN THE SOFTWARE.
31
+
32
+ For commercial or patent licensing enquiries contact: lars@rollik.me
@@ -0,0 +1,226 @@
1
+ Metadata-Version: 2.5
2
+ Name: datapack-neuro
3
+ Version: 1.2.0
4
+ Summary: Lossless and visually-lossless compression of acquisition data (ephys, video, histology) with verification and manifests.
5
+ Project-URL: Homepage, https://github.com/larsrollik/datapack
6
+ Project-URL: Documentation, https://larsrollik.github.io/datapack/
7
+ Project-URL: Issue Tracker, https://github.com/larsrollik/datapack/issues
8
+ Author-email: "Lars B. Rollik" <lars@rollik.me>
9
+ License:
10
+ Copyright (c) 2026-present Lars B. Rollik. All rights reserved.
11
+
12
+ Permission is granted, free of charge, to use, copy, modify, and distribute
13
+ this software and associated documentation files (the "Software") for
14
+ non-commercial research or academic purposes only, subject to the following
15
+ conditions:
16
+
17
+ 1. This copyright notice and permission notice must be included in all copies
18
+ or substantial portions of the Software.
19
+
20
+ 2. Any publication, presentation, or product that uses or builds upon the
21
+ Software must give clear attribution to the original work and its authors.
22
+
23
+ 3. Commercial use is prohibited without a separate written commercial licence
24
+ agreement with the copyright holder. Commercial use means incorporation of
25
+ the Software into anything for which fees or other compensation are charged
26
+ or received, including commercial products and commercial services.
27
+
28
+ 4. No patent licence, express or implied, is granted under these terms. Any
29
+ use that would require a patent licence from the copyright holder requires
30
+ a separate written agreement.
31
+
32
+ 5. Redistribution, in source or binary form, is permitted only for
33
+ non-commercial purposes and must retain this notice unmodified.
34
+
35
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
36
+ IMPLIED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
37
+ DAMAGES, OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THE
38
+ SOFTWARE OR THE USE OR DEALINGS IN THE SOFTWARE.
39
+
40
+ For commercial or patent licensing enquiries contact: lars@rollik.me
41
+ License-File: LICENSE
42
+ Requires-Python: >=3.12
43
+ Requires-Dist: pydantic>=2
44
+ Provides-Extra: dev
45
+ Requires-Dist: commitizen; extra == 'dev'
46
+ Requires-Dist: datapack[ephys]; extra == 'dev'
47
+ Requires-Dist: datapack[histology]; extra == 'dev'
48
+ Requires-Dist: mypy; extra == 'dev'
49
+ Requires-Dist: pre-commit; extra == 'dev'
50
+ Requires-Dist: pytest-cov; extra == 'dev'
51
+ Requires-Dist: pytest>=8; extra == 'dev'
52
+ Provides-Extra: docs
53
+ Requires-Dist: mkdocs-material; extra == 'docs'
54
+ Provides-Extra: ephys
55
+ Requires-Dist: mtscomp==1.0.2; extra == 'ephys'
56
+ Requires-Dist: numpy>=2; extra == 'ephys'
57
+ Requires-Dist: tqdm; extra == 'ephys'
58
+ Provides-Extra: histology
59
+ Requires-Dist: imagecodecs; extra == 'histology'
60
+ Requires-Dist: numpy>=2; extra == 'histology'
61
+ Requires-Dist: tifffile>=2023; extra == 'histology'
62
+ Description-Content-Type: text/markdown
63
+
64
+ # datapack
65
+
66
+ Lossless and visually-lossless compression of acquisition data (ephys, video,
67
+ histology) with verification and manifests.
68
+
69
+ `datapack` moves raw acquisition data into a compressed state, **verifies** every
70
+ artifact, and writes a **manifest** so a downstream offload/tape process can move it
71
+ and delete the original with confidence. It only ever **reads** the source — it never
72
+ mutates or deletes the raw data.
73
+
74
+ ## What it handles
75
+
76
+ | Modality | Lossless? | Engine | Typical saving | Notes |
77
+ |---|---|---|---|---|
78
+ | **Ephys** (Neuropixels / Open Ephys) | **lossless** (bit-exact) | IBL `mtscomp` (`.cbin`+`.ch`) | ~⅓ (NP1 ~59%) | round-trips back to identical `.dat` |
79
+ | **Video** (behavioural, H.264) | **visually** lossless | `ffmpeg` HEVC (NVENC) | ~35–65% | re-encode for DLC/SLEAP; the `.mp4` is the working form |
80
+ | **Histology** (TIFF: brainsaw/StitchIt, brainreg) | **lossless** (bit-exact) | `tifffile` ADOBE_DEFLATE + predictor | ~⅔ (labels 40–75×) | files stay natively readable; tracks/metadata copied verbatim |
81
+
82
+ **Design principles**
83
+
84
+ - **Compress only the large data.** Metadata, `.npy` tracks, recipes, logs, JSON/CSV
85
+ are copied **byte-for-byte verbatim**.
86
+ - **Verify before trusting.** Lossless engines prove a bit-exact round-trip; video
87
+ checks frame-count + clean decode. Verification gates the artifact before it lands.
88
+ - **Round-trip in place.** Lossless outputs stay readable / restorable, so you can work
89
+ on them in scratch (spike-sorting, image processing) and recompress — no lock-in.
90
+ - **Never touch the source.** Write-to-temp → verify → atomic rename; originals untouched.
91
+ - **Manifests** record sizes, ratios, hashes, params and tool versions for the offload step.
92
+
93
+ ## Installation
94
+
95
+ ```sh
96
+ pip install datapack-neuro # core (CLI + histology-independent pieces)
97
+ pip install "datapack-neuro[ephys]" # + mtscomp, numpy, tqdm (ephys engine)
98
+ pip install "datapack-neuro[histology]" # + tifffile, imagecodecs, numpy (histology engine)
99
+ ```
100
+
101
+ The distribution is `datapack-neuro`; the import name is `datapack`. An unrelated
102
+ `datapack` project already holds that name on PyPI.
103
+
104
+ Video needs an **NVENC-enabled `ffmpeg`** on the system (not a Python dep). datapack
105
+ resolves it from `--ffmpeg`, then `$DATAPACK_FFMPEG`, then `PATH`. On a cluster, assume
106
+ the GPU is managed externally — datapack **fails loudly** if no working GPU encoder is
107
+ found rather than silently grinding on CPU (opt into the slow CPU path with `--allow-cpu`).
108
+
109
+ ## CLI
110
+
111
+ Run `datapack <command> --help` for full flags. All commands take a source and a
112
+ destination and never modify the source.
113
+
114
+ ### `compress` — one file, modality auto-detected
115
+ ```sh
116
+ datapack compress <src> <dst> [--modality ephys|video|histology] [--dry-run] [--report manifest.jsonl]
117
+ ```
118
+ Detects the modality from the path (`continuous.dat`→ephys, `.h264/.mp4`→video,
119
+ `.tif/.tiff`→histology) and runs the right engine. For tuned video/ephys runs prefer the
120
+ dedicated commands below.
121
+
122
+ ### Ephys
123
+ ```sh
124
+ # Compress a whole Open Ephys session: every continuous.dat -> .cbin/.ch,
125
+ # all sidecars (structure.oebin, *.npy, events/, settings.xml ...) copied verbatim.
126
+ datapack oe-session <session_dir> <dest_dir> [--threads N] [--chunk-duration S] \
127
+ [--no-sidecars] [--overwrite] [--dry-run] [--report manifest.jsonl]
128
+
129
+ # Restore a compressed session to a working/scratch tree (bit-exact .dat) for
130
+ # SpikeInterface / Kilosort. Inverse of oe-session.
131
+ datapack oe-restore <compressed_session_dir> <work_dir> [--overwrite]
132
+ ```
133
+
134
+ ### Video
135
+ ```sh
136
+ # Visually-lossless HEVC re-encode (NVENC; libx265 CPU fallback only with --allow-cpu).
137
+ datapack video <src.h264> <dst.mp4> [--quality safe|balanced|small] [--qp N] [--fps F] \
138
+ [--ffmpeg /path/to/ffmpeg] [--allow-cpu] [--report manifest.jsonl]
139
+ ```
140
+ `--quality` presets (preset p7, GOP 100): `safe`=qp18, `balanced`=qp20 (default),
141
+ `small`=qp22. **Provisional** pending the lab's DLC/SLEAP tracking validation; `--qp`
142
+ overrides. Raw `.h264` has no timing, so `--fps` (default 100) is forced on input.
143
+
144
+ ### Histology
145
+ ```sh
146
+ # In-place lossless recompression of every TIFF in a tree (e.g. a brainreg
147
+ # registration/), copying non-TIFF (incl. brainreg .npy tracks) verbatim. Output
148
+ # files stay normal, natively-readable TIFFs (ADOBE_DEFLATE) — no unpack needed.
149
+ datapack histology tree <src_dir> <dst_dir> [--codec deflate|zstd] [--level N] \
150
+ [--no-tile] [--overwrite] [--dry-run] [--report manifest.jsonl]
151
+
152
+ # Optional bundling: recompress a TIFF dir then store it as ONE archive (adds no
153
+ # compression over `tree` — purely bundling for transfer/tape).
154
+ datapack histology pack <tiff_dir> <archive.zip> [--codec ...] [--level N] [--overwrite] [--dry-run]
155
+ datapack histology unpack <archive.zip> <work_dir> [--no-verify] # extract + re-verify
156
+ datapack histology repack <work_dir> <archive.zip> [--codec ...] [--level N] # after editing
157
+ ```
158
+
159
+ ### Datasets (dataset-aware "pack and stash")
160
+ ```sh
161
+ # Pack a brainsaw/StitchIt serial_section/<sample>/ sample. Default: recompress every
162
+ # TIFF in place (raw + registration) -> stays browsable; copy .npy tracks + recipe/
163
+ # logs/mat/ini verbatim. Use --archive-raw to bundle stitchedImages/downsampled
164
+ # into one zip each (no ratio benefit, only one-object-per-dir for transfer/tape).
165
+ datapack serial-section <sample_dir> <dest_dir> [--archive-raw] [--codec deflate|zstd] \
166
+ [--overwrite] [--dry-run]
167
+ ```
168
+
169
+ ## Python API
170
+
171
+ ```python
172
+ import datapack
173
+
174
+ # one file (modality auto-detected or forced)
175
+ rec = datapack.compress_file("continuous.dat", "out.cbin",
176
+ params={"sample_rate": 30000, "n_channels": 384})
177
+ print(rec.ratio, rec.lossless, rec.verification.ok, rec.ready_to_offload)
178
+
179
+ # ephys session round-trip
180
+ datapack.compress_oe_session("session/", "compressed/") # -> .cbin/.ch + sidecars
181
+ datapack.decompress_oe_session("compressed/", "scratch/") # -> bit-exact .dat tree
182
+
183
+ # dataset-aware
184
+ datapack.pack_serial_section("serial_section/ap5l/", "packed/")
185
+
186
+ from datapack.engines import (
187
+ EphysEngine, HistologyEngine, VideoEngine, # Engine.compress(Job) -> ManifestRecord
188
+ compress_tiff_tree, pack, unpack, repack, # histology tree + archive
189
+ probe_ffmpeg, # video capability probe
190
+ )
191
+ from datapack.core import Job, ManifestRecord, ManifestWriter, read_manifest
192
+ ```
193
+
194
+ Every engine returns a `ManifestRecord` (pydantic): source/output paths + sizes, ratio,
195
+ `lossless`, the `VerifyResult`, engine + tool versions, and engine-specific `extra` (e.g.
196
+ mtscomp `sha1_uncompressed`/`sha1_compressed`, or a TIFF decoded-pixel SHA-256). Write a
197
+ run's records to JSON-lines with `ManifestWriter` and read them back with `read_manifest`.
198
+
199
+ ## Development setup
200
+
201
+ ```sh
202
+ git clone https://github.com/larsrollik/datapack.git
203
+ cd datapack
204
+ uv sync --extra dev
205
+ uv run pre-commit install --hook-type pre-commit --hook-type commit-msg
206
+ ```
207
+
208
+ ## Running tests
209
+
210
+ ```sh
211
+ uv run pytest
212
+ ```
213
+
214
+ Tests needing optional deps or a GPU skip automatically when unavailable. To exercise the
215
+ real NVENC video path, point `$DATAPACK_FFMPEG` at an nvenc-enabled `ffmpeg`.
216
+
217
+ ## Release workflow
218
+
219
+ 1. Work on a `feature/` or `fix/` branch, committing with `cz commit`
220
+ 2. Open a PR — CI (lint + tests + secrets scan) must pass before merge
221
+ 3. Merge to main → version bump and tag are created automatically
222
+ 4. Tag triggers release: GitHub release + PyPI publish
223
+
224
+ ## License
225
+
226
+ See [LICENSE](LICENSE).
@@ -0,0 +1,163 @@
1
+ # datapack
2
+
3
+ Lossless and visually-lossless compression of acquisition data (ephys, video,
4
+ histology) with verification and manifests.
5
+
6
+ `datapack` moves raw acquisition data into a compressed state, **verifies** every
7
+ artifact, and writes a **manifest** so a downstream offload/tape process can move it
8
+ and delete the original with confidence. It only ever **reads** the source — it never
9
+ mutates or deletes the raw data.
10
+
11
+ ## What it handles
12
+
13
+ | Modality | Lossless? | Engine | Typical saving | Notes |
14
+ |---|---|---|---|---|
15
+ | **Ephys** (Neuropixels / Open Ephys) | **lossless** (bit-exact) | IBL `mtscomp` (`.cbin`+`.ch`) | ~⅓ (NP1 ~59%) | round-trips back to identical `.dat` |
16
+ | **Video** (behavioural, H.264) | **visually** lossless | `ffmpeg` HEVC (NVENC) | ~35–65% | re-encode for DLC/SLEAP; the `.mp4` is the working form |
17
+ | **Histology** (TIFF: brainsaw/StitchIt, brainreg) | **lossless** (bit-exact) | `tifffile` ADOBE_DEFLATE + predictor | ~⅔ (labels 40–75×) | files stay natively readable; tracks/metadata copied verbatim |
18
+
19
+ **Design principles**
20
+
21
+ - **Compress only the large data.** Metadata, `.npy` tracks, recipes, logs, JSON/CSV
22
+ are copied **byte-for-byte verbatim**.
23
+ - **Verify before trusting.** Lossless engines prove a bit-exact round-trip; video
24
+ checks frame-count + clean decode. Verification gates the artifact before it lands.
25
+ - **Round-trip in place.** Lossless outputs stay readable / restorable, so you can work
26
+ on them in scratch (spike-sorting, image processing) and recompress — no lock-in.
27
+ - **Never touch the source.** Write-to-temp → verify → atomic rename; originals untouched.
28
+ - **Manifests** record sizes, ratios, hashes, params and tool versions for the offload step.
29
+
30
+ ## Installation
31
+
32
+ ```sh
33
+ pip install datapack-neuro # core (CLI + histology-independent pieces)
34
+ pip install "datapack-neuro[ephys]" # + mtscomp, numpy, tqdm (ephys engine)
35
+ pip install "datapack-neuro[histology]" # + tifffile, imagecodecs, numpy (histology engine)
36
+ ```
37
+
38
+ The distribution is `datapack-neuro`; the import name is `datapack`. An unrelated
39
+ `datapack` project already holds that name on PyPI.
40
+
41
+ Video needs an **NVENC-enabled `ffmpeg`** on the system (not a Python dep). datapack
42
+ resolves it from `--ffmpeg`, then `$DATAPACK_FFMPEG`, then `PATH`. On a cluster, assume
43
+ the GPU is managed externally — datapack **fails loudly** if no working GPU encoder is
44
+ found rather than silently grinding on CPU (opt into the slow CPU path with `--allow-cpu`).
45
+
46
+ ## CLI
47
+
48
+ Run `datapack <command> --help` for full flags. All commands take a source and a
49
+ destination and never modify the source.
50
+
51
+ ### `compress` — one file, modality auto-detected
52
+ ```sh
53
+ datapack compress <src> <dst> [--modality ephys|video|histology] [--dry-run] [--report manifest.jsonl]
54
+ ```
55
+ Detects the modality from the path (`continuous.dat`→ephys, `.h264/.mp4`→video,
56
+ `.tif/.tiff`→histology) and runs the right engine. For tuned video/ephys runs prefer the
57
+ dedicated commands below.
58
+
59
+ ### Ephys
60
+ ```sh
61
+ # Compress a whole Open Ephys session: every continuous.dat -> .cbin/.ch,
62
+ # all sidecars (structure.oebin, *.npy, events/, settings.xml ...) copied verbatim.
63
+ datapack oe-session <session_dir> <dest_dir> [--threads N] [--chunk-duration S] \
64
+ [--no-sidecars] [--overwrite] [--dry-run] [--report manifest.jsonl]
65
+
66
+ # Restore a compressed session to a working/scratch tree (bit-exact .dat) for
67
+ # SpikeInterface / Kilosort. Inverse of oe-session.
68
+ datapack oe-restore <compressed_session_dir> <work_dir> [--overwrite]
69
+ ```
70
+
71
+ ### Video
72
+ ```sh
73
+ # Visually-lossless HEVC re-encode (NVENC; libx265 CPU fallback only with --allow-cpu).
74
+ datapack video <src.h264> <dst.mp4> [--quality safe|balanced|small] [--qp N] [--fps F] \
75
+ [--ffmpeg /path/to/ffmpeg] [--allow-cpu] [--report manifest.jsonl]
76
+ ```
77
+ `--quality` presets (preset p7, GOP 100): `safe`=qp18, `balanced`=qp20 (default),
78
+ `small`=qp22. **Provisional** pending the lab's DLC/SLEAP tracking validation; `--qp`
79
+ overrides. Raw `.h264` has no timing, so `--fps` (default 100) is forced on input.
80
+
81
+ ### Histology
82
+ ```sh
83
+ # In-place lossless recompression of every TIFF in a tree (e.g. a brainreg
84
+ # registration/), copying non-TIFF (incl. brainreg .npy tracks) verbatim. Output
85
+ # files stay normal, natively-readable TIFFs (ADOBE_DEFLATE) — no unpack needed.
86
+ datapack histology tree <src_dir> <dst_dir> [--codec deflate|zstd] [--level N] \
87
+ [--no-tile] [--overwrite] [--dry-run] [--report manifest.jsonl]
88
+
89
+ # Optional bundling: recompress a TIFF dir then store it as ONE archive (adds no
90
+ # compression over `tree` — purely bundling for transfer/tape).
91
+ datapack histology pack <tiff_dir> <archive.zip> [--codec ...] [--level N] [--overwrite] [--dry-run]
92
+ datapack histology unpack <archive.zip> <work_dir> [--no-verify] # extract + re-verify
93
+ datapack histology repack <work_dir> <archive.zip> [--codec ...] [--level N] # after editing
94
+ ```
95
+
96
+ ### Datasets (dataset-aware "pack and stash")
97
+ ```sh
98
+ # Pack a brainsaw/StitchIt serial_section/<sample>/ sample. Default: recompress every
99
+ # TIFF in place (raw + registration) -> stays browsable; copy .npy tracks + recipe/
100
+ # logs/mat/ini verbatim. Use --archive-raw to bundle stitchedImages/downsampled
101
+ # into one zip each (no ratio benefit, only one-object-per-dir for transfer/tape).
102
+ datapack serial-section <sample_dir> <dest_dir> [--archive-raw] [--codec deflate|zstd] \
103
+ [--overwrite] [--dry-run]
104
+ ```
105
+
106
+ ## Python API
107
+
108
+ ```python
109
+ import datapack
110
+
111
+ # one file (modality auto-detected or forced)
112
+ rec = datapack.compress_file("continuous.dat", "out.cbin",
113
+ params={"sample_rate": 30000, "n_channels": 384})
114
+ print(rec.ratio, rec.lossless, rec.verification.ok, rec.ready_to_offload)
115
+
116
+ # ephys session round-trip
117
+ datapack.compress_oe_session("session/", "compressed/") # -> .cbin/.ch + sidecars
118
+ datapack.decompress_oe_session("compressed/", "scratch/") # -> bit-exact .dat tree
119
+
120
+ # dataset-aware
121
+ datapack.pack_serial_section("serial_section/ap5l/", "packed/")
122
+
123
+ from datapack.engines import (
124
+ EphysEngine, HistologyEngine, VideoEngine, # Engine.compress(Job) -> ManifestRecord
125
+ compress_tiff_tree, pack, unpack, repack, # histology tree + archive
126
+ probe_ffmpeg, # video capability probe
127
+ )
128
+ from datapack.core import Job, ManifestRecord, ManifestWriter, read_manifest
129
+ ```
130
+
131
+ Every engine returns a `ManifestRecord` (pydantic): source/output paths + sizes, ratio,
132
+ `lossless`, the `VerifyResult`, engine + tool versions, and engine-specific `extra` (e.g.
133
+ mtscomp `sha1_uncompressed`/`sha1_compressed`, or a TIFF decoded-pixel SHA-256). Write a
134
+ run's records to JSON-lines with `ManifestWriter` and read them back with `read_manifest`.
135
+
136
+ ## Development setup
137
+
138
+ ```sh
139
+ git clone https://github.com/larsrollik/datapack.git
140
+ cd datapack
141
+ uv sync --extra dev
142
+ uv run pre-commit install --hook-type pre-commit --hook-type commit-msg
143
+ ```
144
+
145
+ ## Running tests
146
+
147
+ ```sh
148
+ uv run pytest
149
+ ```
150
+
151
+ Tests needing optional deps or a GPU skip automatically when unavailable. To exercise the
152
+ real NVENC video path, point `$DATAPACK_FFMPEG` at an nvenc-enabled `ffmpeg`.
153
+
154
+ ## Release workflow
155
+
156
+ 1. Work on a `feature/` or `fix/` branch, committing with `cz commit`
157
+ 2. Open a PR — CI (lint + tests + secrets scan) must pass before merge
158
+ 3. Merge to main → version bump and tag are created automatically
159
+ 4. Tag triggers release: GitHub release + PyPI publish
160
+
161
+ ## License
162
+
163
+ See [LICENSE](LICENSE).
@@ -0,0 +1 @@
1
+ 1.2.0
@@ -0,0 +1,154 @@
1
+ [build-system]
2
+ requires = ["hatchling", "hatch-vcs"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "datapack-neuro"
7
+ dynamic = ["version"]
8
+ description = "Lossless and visually-lossless compression of acquisition data (ephys, video, histology) with verification and manifests."
9
+ readme = { file = "README.md", content-type = "text/markdown" }
10
+ license = { file = "LICENSE" }
11
+ authors = [
12
+ { name = "Lars B. Rollik", email = "lars@rollik.me" },
13
+ ]
14
+ requires-python = ">=3.12"
15
+ dependencies = [
16
+ "pydantic>=2", # manifest + config models
17
+ ]
18
+
19
+ [project.scripts]
20
+ datapack = "datapack.cli:main"
21
+
22
+ [project.urls]
23
+ Homepage = "https://github.com/larsrollik/datapack"
24
+ Documentation = "https://larsrollik.github.io/datapack/"
25
+ "Issue Tracker" = "https://github.com/larsrollik/datapack/issues"
26
+
27
+ # ---------------------------------------------------------------------------
28
+ # Build
29
+ # ---------------------------------------------------------------------------
30
+
31
+ [tool.hatch.version]
32
+ source = "vcs"
33
+ fallback-version = "1.0.0"
34
+
35
+ [tool.hatch.build.hooks.vcs]
36
+ version-file = "src/datapack/_version.py"
37
+
38
+ [tool.hatch.build.targets.wheel]
39
+ packages = ["src/datapack"]
40
+
41
+ [tool.hatch.build.targets.sdist]
42
+ include = [
43
+ "/src/datapack",
44
+ "/tests",
45
+ "/README.md",
46
+ "/LICENSE",
47
+ "/pyproject.toml",
48
+ "/CITATION.cff",
49
+ "/VERSION",
50
+ ]
51
+
52
+ # ---------------------------------------------------------------------------
53
+ # Dependencies
54
+ # ---------------------------------------------------------------------------
55
+
56
+ [project.optional-dependencies]
57
+ ephys = [
58
+ "mtscomp==1.0.2", # IBL lossless ephys compression (.cbin/.ch); see ROADMAP D1/D6
59
+ "numpy>=2", # mtscomp runtime dep; numpy 2.x verified
60
+ "tqdm", # mtscomp imports it but its wheel does NOT declare it
61
+ ]
62
+ histology = [
63
+ "tifffile>=2023", # lossless TIFF read/write (BigTIFF, tiled, predictor)
64
+ "imagecodecs", # codec backends (deflate, zstd) — manylinux wheels, no system libs
65
+ "numpy>=2", # pixel-array verification
66
+ ]
67
+ dev = [
68
+ "commitizen",
69
+ "pytest>=8",
70
+ "pytest-cov",
71
+ "pre-commit",
72
+ "mypy",
73
+ "datapack[ephys]", # so the ephys engine is exercised in tests/CI
74
+ "datapack[histology]", # so the histology engine + archive are exercised in tests/CI
75
+ ]
76
+ docs = [
77
+ "mkdocs-material",
78
+ # Uncomment to enable auto-generated API reference from docstrings:
79
+ # "mkdocstrings[python]",
80
+ ]
81
+
82
+ # ---------------------------------------------------------------------------
83
+ # Commitizen
84
+ # ---------------------------------------------------------------------------
85
+
86
+ [tool.commitizen]
87
+ name = "cz_conventional_commits"
88
+ version_provider = "commitizen"
89
+ version = "1.2.0"
90
+ version_files = ["VERSION"]
91
+ tag_format = "v$version"
92
+ update_changelog_on_bump = false
93
+
94
+ # ---------------------------------------------------------------------------
95
+ # Pytest
96
+ # ---------------------------------------------------------------------------
97
+
98
+ [tool.pytest.ini_options]
99
+ testpaths = ["tests"]
100
+ addopts = "--cov=src/datapack --cov-report=term-missing --maxfail=5 --import-mode=importlib"
101
+
102
+ # ---------------------------------------------------------------------------
103
+ # Ruff
104
+ # ---------------------------------------------------------------------------
105
+
106
+ [tool.ruff]
107
+ line-length = 88
108
+ indent-width = 4
109
+ src = ["src"]
110
+
111
+ [tool.ruff.lint]
112
+ select = [
113
+ "E", "W", # pycodestyle
114
+ "F", # pyflakes
115
+ "I", # isort
116
+ "UP", # pyupgrade
117
+ "B", # flake8-bugbear
118
+ "SIM", # flake8-simplify
119
+ "PTH", # flake8-pathlib
120
+ "TCH", # flake8-type-checking
121
+ "PYI", # flake8-pyi
122
+ "YTT", # flake8-2020
123
+ "N", # pep8-naming
124
+ ]
125
+ ignore = [
126
+ "N818", # keep expressive exception names (EngineNotAvailable) without an Error suffix
127
+ ]
128
+ fixable = ["ALL"]
129
+
130
+ [tool.ruff.format]
131
+ quote-style = "double"
132
+ indent-style = "space"
133
+ docstring-code-format = true
134
+
135
+ [tool.ruff.lint.pydocstyle]
136
+ convention = "numpy"
137
+
138
+ [tool.ruff.lint.flake8-type-checking]
139
+ # pydantic models evaluate annotations at runtime, so field-type imports must
140
+ # stay at module scope — don't let TC rules push them into TYPE_CHECKING blocks.
141
+ runtime-evaluated-base-classes = ["pydantic.BaseModel"]
142
+
143
+ [tool.ruff.lint.per-file-ignores]
144
+ "tests/**" = ["TC001", "TC002", "TC003"] # annotation-only imports are fine in tests
145
+
146
+ # ---------------------------------------------------------------------------
147
+ # Mypy
148
+ # ---------------------------------------------------------------------------
149
+
150
+ [tool.mypy]
151
+ python_version = "3.12"
152
+ warn_return_any = true
153
+ warn_unused_ignores = true
154
+ ignore_missing_imports = true