datapack-neuro 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datapack_neuro-1.2.0/.gitignore +71 -0
- datapack_neuro-1.2.0/CITATION.cff +37 -0
- datapack_neuro-1.2.0/LICENSE +32 -0
- datapack_neuro-1.2.0/PKG-INFO +226 -0
- datapack_neuro-1.2.0/README.md +163 -0
- datapack_neuro-1.2.0/VERSION +1 -0
- datapack_neuro-1.2.0/pyproject.toml +154 -0
- datapack_neuro-1.2.0/src/datapack/__init__.py +124 -0
- datapack_neuro-1.2.0/src/datapack/_version.py +24 -0
- datapack_neuro-1.2.0/src/datapack/cli.py +379 -0
- datapack_neuro-1.2.0/src/datapack/config.py +62 -0
- datapack_neuro-1.2.0/src/datapack/core/__init__.py +41 -0
- datapack_neuro-1.2.0/src/datapack/core/atomic.py +90 -0
- datapack_neuro-1.2.0/src/datapack/core/detect.py +55 -0
- datapack_neuro-1.2.0/src/datapack/core/engine.py +39 -0
- datapack_neuro-1.2.0/src/datapack/core/errors.py +40 -0
- datapack_neuro-1.2.0/src/datapack/core/job.py +37 -0
- datapack_neuro-1.2.0/src/datapack/core/manifest.py +83 -0
- datapack_neuro-1.2.0/src/datapack/core/verify.py +45 -0
- datapack_neuro-1.2.0/src/datapack/datasets.py +143 -0
- datapack_neuro-1.2.0/src/datapack/engines/__init__.py +53 -0
- datapack_neuro-1.2.0/src/datapack/engines/ephys.py +595 -0
- datapack_neuro-1.2.0/src/datapack/engines/histology.py +584 -0
- datapack_neuro-1.2.0/src/datapack/engines/video.py +433 -0
- datapack_neuro-1.2.0/src/datapack/py.typed +0 -0
- datapack_neuro-1.2.0/tests/conftest.py +1 -0
- datapack_neuro-1.2.0/tests/integration/test_ephys_roundtrip.py +92 -0
- datapack_neuro-1.2.0/tests/integration/test_histology.py +210 -0
- datapack_neuro-1.2.0/tests/integration/test_oe_session.py +227 -0
- datapack_neuro-1.2.0/tests/integration/test_placeholder.py +6 -0
- datapack_neuro-1.2.0/tests/integration/test_serial_section.py +116 -0
- datapack_neuro-1.2.0/tests/integration/test_video_encode.py +61 -0
- datapack_neuro-1.2.0/tests/unit/test_cli.py +53 -0
- datapack_neuro-1.2.0/tests/unit/test_core.py +219 -0
- datapack_neuro-1.2.0/tests/unit/test_ephys.py +78 -0
- datapack_neuro-1.2.0/tests/unit/test_video.py +117 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.pyo
|
|
5
|
+
*.pyd
|
|
6
|
+
*.so
|
|
7
|
+
*.egg
|
|
8
|
+
*.egg-info/
|
|
9
|
+
dist/
|
|
10
|
+
build/
|
|
11
|
+
eggs/
|
|
12
|
+
parts/
|
|
13
|
+
var/
|
|
14
|
+
sdist/
|
|
15
|
+
develop-eggs/
|
|
16
|
+
.installed.cfg
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
wheels/
|
|
20
|
+
|
|
21
|
+
# Virtual environments
|
|
22
|
+
.venv/
|
|
23
|
+
venv/
|
|
24
|
+
env/
|
|
25
|
+
ENV/
|
|
26
|
+
.python-version
|
|
27
|
+
|
|
28
|
+
# uv
|
|
29
|
+
uv.lock
|
|
30
|
+
|
|
31
|
+
# Distribution / packaging
|
|
32
|
+
MANIFEST
|
|
33
|
+
|
|
34
|
+
# Testing
|
|
35
|
+
.pytest_cache/
|
|
36
|
+
.coverage
|
|
37
|
+
.coverage.*
|
|
38
|
+
coverage.xml
|
|
39
|
+
htmlcov/
|
|
40
|
+
*.cover
|
|
41
|
+
.hypothesis/
|
|
42
|
+
|
|
43
|
+
# Type checking
|
|
44
|
+
.mypy_cache/
|
|
45
|
+
.dmypy.json
|
|
46
|
+
dmypy.json
|
|
47
|
+
.pytype/
|
|
48
|
+
.pyre/
|
|
49
|
+
|
|
50
|
+
# Build artifacts (hatch-vcs generated)
|
|
51
|
+
src/*/_version.py
|
|
52
|
+
|
|
53
|
+
# Jupyter
|
|
54
|
+
.ipynb_checkpoints
|
|
55
|
+
*.ipynb
|
|
56
|
+
|
|
57
|
+
# IDEs
|
|
58
|
+
.idea/
|
|
59
|
+
.vscode/
|
|
60
|
+
*.swp
|
|
61
|
+
*.swo
|
|
62
|
+
*~
|
|
63
|
+
|
|
64
|
+
# OS
|
|
65
|
+
.DS_Store
|
|
66
|
+
Thumbs.db
|
|
67
|
+
|
|
68
|
+
# Secrets / local config
|
|
69
|
+
.env
|
|
70
|
+
.env.*
|
|
71
|
+
!.env.example
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
type: software
|
|
4
|
+
title: "datapack"
|
|
5
|
+
version: "1.1.0"
|
|
6
|
+
repository-code: https://github.com/larsrollik/datapack
|
|
7
|
+
url: https://github.com/larsrollik/datapack
|
|
8
|
+
# Not an SPDX identifier: the terms are custom (non-commercial research use only),
|
|
9
|
+
# so CFF takes a URL rather than an id. Declaring GPL-3.0 here, as this file used to,
|
|
10
|
+
# contradicted LICENSE and would have told a reader they could use this commercially.
|
|
11
|
+
license-url: https://github.com/larsrollik/datapack/blob/main/LICENSE
|
|
12
|
+
authors:
|
|
13
|
+
- family-names: Rollik
|
|
14
|
+
given-names: Lars
|
|
15
|
+
# orcid: add before the first tagged release; a placeholder id must not ship
|
|
16
|
+
|
|
17
|
+
# Zenodo integration (optional):
|
|
18
|
+
# 1. Enable the Zenodo webhook at zenodo.org/account/settings/github
|
|
19
|
+
# 2. After first tag release, copy the concept DOI from Zenodo and add:
|
|
20
|
+
#
|
|
21
|
+
# identifiers:
|
|
22
|
+
# - type: doi
|
|
23
|
+
# value: 10.5281/zenodo.XXXXXXX
|
|
24
|
+
# description: Zenodo concept DOI
|
|
25
|
+
#
|
|
26
|
+
# preferred-citation:
|
|
27
|
+
# type: software
|
|
28
|
+
# title: "datapack"
|
|
29
|
+
# authors:
|
|
30
|
+
# - family-names: ...
|
|
31
|
+
# given-names: ...
|
|
32
|
+
# orcid: https://orcid.org/...
|
|
33
|
+
# year: 2026
|
|
34
|
+
# publisher:
|
|
35
|
+
# name: Zenodo
|
|
36
|
+
# doi: 10.5281/zenodo.XXXXXXX
|
|
37
|
+
# url: https://doi.org/10.5281/zenodo.XXXXXXX
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
|
|
2
|
+
Copyright (c) 2026-present Lars B. Rollik. All rights reserved.
|
|
3
|
+
|
|
4
|
+
Permission is granted, free of charge, to use, copy, modify, and distribute
|
|
5
|
+
this software and associated documentation files (the "Software") for
|
|
6
|
+
non-commercial research or academic purposes only, subject to the following
|
|
7
|
+
conditions:
|
|
8
|
+
|
|
9
|
+
1. This copyright notice and permission notice must be included in all copies
|
|
10
|
+
or substantial portions of the Software.
|
|
11
|
+
|
|
12
|
+
2. Any publication, presentation, or product that uses or builds upon the
|
|
13
|
+
Software must give clear attribution to the original work and its authors.
|
|
14
|
+
|
|
15
|
+
3. Commercial use is prohibited without a separate written commercial licence
|
|
16
|
+
agreement with the copyright holder. Commercial use means incorporation of
|
|
17
|
+
the Software into anything for which fees or other compensation are charged
|
|
18
|
+
or received, including commercial products and commercial services.
|
|
19
|
+
|
|
20
|
+
4. No patent licence, express or implied, is granted under these terms. Any
|
|
21
|
+
use that would require a patent licence from the copyright holder requires
|
|
22
|
+
a separate written agreement.
|
|
23
|
+
|
|
24
|
+
5. Redistribution, in source or binary form, is permitted only for
|
|
25
|
+
non-commercial purposes and must retain this notice unmodified.
|
|
26
|
+
|
|
27
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
28
|
+
IMPLIED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
|
|
29
|
+
DAMAGES, OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THE
|
|
30
|
+
SOFTWARE OR THE USE OR DEALINGS IN THE SOFTWARE.
|
|
31
|
+
|
|
32
|
+
For commercial or patent licensing enquiries contact: lars@rollik.me
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: datapack-neuro
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Lossless and visually-lossless compression of acquisition data (ephys, video, histology) with verification and manifests.
|
|
5
|
+
Project-URL: Homepage, https://github.com/larsrollik/datapack
|
|
6
|
+
Project-URL: Documentation, https://larsrollik.github.io/datapack/
|
|
7
|
+
Project-URL: Issue Tracker, https://github.com/larsrollik/datapack/issues
|
|
8
|
+
Author-email: "Lars B. Rollik" <lars@rollik.me>
|
|
9
|
+
License:
|
|
10
|
+
Copyright (c) 2026-present Lars B. Rollik. All rights reserved.
|
|
11
|
+
|
|
12
|
+
Permission is granted, free of charge, to use, copy, modify, and distribute
|
|
13
|
+
this software and associated documentation files (the "Software") for
|
|
14
|
+
non-commercial research or academic purposes only, subject to the following
|
|
15
|
+
conditions:
|
|
16
|
+
|
|
17
|
+
1. This copyright notice and permission notice must be included in all copies
|
|
18
|
+
or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
2. Any publication, presentation, or product that uses or builds upon the
|
|
21
|
+
Software must give clear attribution to the original work and its authors.
|
|
22
|
+
|
|
23
|
+
3. Commercial use is prohibited without a separate written commercial licence
|
|
24
|
+
agreement with the copyright holder. Commercial use means incorporation of
|
|
25
|
+
the Software into anything for which fees or other compensation are charged
|
|
26
|
+
or received, including commercial products and commercial services.
|
|
27
|
+
|
|
28
|
+
4. No patent licence, express or implied, is granted under these terms. Any
|
|
29
|
+
use that would require a patent licence from the copyright holder requires
|
|
30
|
+
a separate written agreement.
|
|
31
|
+
|
|
32
|
+
5. Redistribution, in source or binary form, is permitted only for
|
|
33
|
+
non-commercial purposes and must retain this notice unmodified.
|
|
34
|
+
|
|
35
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
36
|
+
IMPLIED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM,
|
|
37
|
+
DAMAGES, OR OTHER LIABILITY ARISING FROM, OUT OF, OR IN CONNECTION WITH THE
|
|
38
|
+
SOFTWARE OR THE USE OR DEALINGS IN THE SOFTWARE.
|
|
39
|
+
|
|
40
|
+
For commercial or patent licensing enquiries contact: lars@rollik.me
|
|
41
|
+
License-File: LICENSE
|
|
42
|
+
Requires-Python: >=3.12
|
|
43
|
+
Requires-Dist: pydantic>=2
|
|
44
|
+
Provides-Extra: dev
|
|
45
|
+
Requires-Dist: commitizen; extra == 'dev'
|
|
46
|
+
Requires-Dist: datapack[ephys]; extra == 'dev'
|
|
47
|
+
Requires-Dist: datapack[histology]; extra == 'dev'
|
|
48
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
49
|
+
Requires-Dist: pre-commit; extra == 'dev'
|
|
50
|
+
Requires-Dist: pytest-cov; extra == 'dev'
|
|
51
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
52
|
+
Provides-Extra: docs
|
|
53
|
+
Requires-Dist: mkdocs-material; extra == 'docs'
|
|
54
|
+
Provides-Extra: ephys
|
|
55
|
+
Requires-Dist: mtscomp==1.0.2; extra == 'ephys'
|
|
56
|
+
Requires-Dist: numpy>=2; extra == 'ephys'
|
|
57
|
+
Requires-Dist: tqdm; extra == 'ephys'
|
|
58
|
+
Provides-Extra: histology
|
|
59
|
+
Requires-Dist: imagecodecs; extra == 'histology'
|
|
60
|
+
Requires-Dist: numpy>=2; extra == 'histology'
|
|
61
|
+
Requires-Dist: tifffile>=2023; extra == 'histology'
|
|
62
|
+
Description-Content-Type: text/markdown
|
|
63
|
+
|
|
64
|
+
# datapack
|
|
65
|
+
|
|
66
|
+
Lossless and visually-lossless compression of acquisition data (ephys, video,
|
|
67
|
+
histology) with verification and manifests.
|
|
68
|
+
|
|
69
|
+
`datapack` moves raw acquisition data into a compressed state, **verifies** every
|
|
70
|
+
artifact, and writes a **manifest** so a downstream offload/tape process can move it
|
|
71
|
+
and delete the original with confidence. It only ever **reads** the source — it never
|
|
72
|
+
mutates or deletes the raw data.
|
|
73
|
+
|
|
74
|
+
## What it handles
|
|
75
|
+
|
|
76
|
+
| Modality | Lossless? | Engine | Typical saving | Notes |
|
|
77
|
+
|---|---|---|---|---|
|
|
78
|
+
| **Ephys** (Neuropixels / Open Ephys) | **lossless** (bit-exact) | IBL `mtscomp` (`.cbin`+`.ch`) | ~⅓ (NP1 ~59%) | round-trips back to identical `.dat` |
|
|
79
|
+
| **Video** (behavioural, H.264) | **visually** lossless | `ffmpeg` HEVC (NVENC) | ~35–65% | re-encode for DLC/SLEAP; the `.mp4` is the working form |
|
|
80
|
+
| **Histology** (TIFF: brainsaw/StitchIt, brainreg) | **lossless** (bit-exact) | `tifffile` ADOBE_DEFLATE + predictor | ~⅔ (labels 40–75×) | files stay natively readable; tracks/metadata copied verbatim |
|
|
81
|
+
|
|
82
|
+
**Design principles**
|
|
83
|
+
|
|
84
|
+
- **Compress only the large data.** Metadata, `.npy` tracks, recipes, logs, JSON/CSV
|
|
85
|
+
are copied **byte-for-byte verbatim**.
|
|
86
|
+
- **Verify before trusting.** Lossless engines prove a bit-exact round-trip; video
|
|
87
|
+
checks frame-count + clean decode. Verification gates the artifact before it lands.
|
|
88
|
+
- **Round-trip in place.** Lossless outputs stay readable / restorable, so you can work
|
|
89
|
+
on them in scratch (spike-sorting, image processing) and recompress — no lock-in.
|
|
90
|
+
- **Never touch the source.** Write-to-temp → verify → atomic rename; originals untouched.
|
|
91
|
+
- **Manifests** record sizes, ratios, hashes, params and tool versions for the offload step.
|
|
92
|
+
|
|
93
|
+
## Installation
|
|
94
|
+
|
|
95
|
+
```sh
|
|
96
|
+
pip install datapack-neuro # core (CLI + histology-independent pieces)
|
|
97
|
+
pip install "datapack-neuro[ephys]" # + mtscomp, numpy, tqdm (ephys engine)
|
|
98
|
+
pip install "datapack-neuro[histology]" # + tifffile, imagecodecs, numpy (histology engine)
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
The distribution is `datapack-neuro`; the import name is `datapack`. An unrelated
|
|
102
|
+
`datapack` project already holds that name on PyPI.
|
|
103
|
+
|
|
104
|
+
Video needs an **NVENC-enabled `ffmpeg`** on the system (not a Python dep). datapack
|
|
105
|
+
resolves it from `--ffmpeg`, then `$DATAPACK_FFMPEG`, then `PATH`. On a cluster, assume
|
|
106
|
+
the GPU is managed externally — datapack **fails loudly** if no working GPU encoder is
|
|
107
|
+
found rather than silently grinding on CPU (opt into the slow CPU path with `--allow-cpu`).
|
|
108
|
+
|
|
109
|
+
## CLI
|
|
110
|
+
|
|
111
|
+
Run `datapack <command> --help` for full flags. All commands take a source and a
|
|
112
|
+
destination and never modify the source.
|
|
113
|
+
|
|
114
|
+
### `compress` — one file, modality auto-detected
|
|
115
|
+
```sh
|
|
116
|
+
datapack compress <src> <dst> [--modality ephys|video|histology] [--dry-run] [--report manifest.jsonl]
|
|
117
|
+
```
|
|
118
|
+
Detects the modality from the path (`continuous.dat`→ephys, `.h264/.mp4`→video,
|
|
119
|
+
`.tif/.tiff`→histology) and runs the right engine. For tuned video/ephys runs prefer the
|
|
120
|
+
dedicated commands below.
|
|
121
|
+
|
|
122
|
+
### Ephys
|
|
123
|
+
```sh
|
|
124
|
+
# Compress a whole Open Ephys session: every continuous.dat -> .cbin/.ch,
|
|
125
|
+
# all sidecars (structure.oebin, *.npy, events/, settings.xml ...) copied verbatim.
|
|
126
|
+
datapack oe-session <session_dir> <dest_dir> [--threads N] [--chunk-duration S] \
|
|
127
|
+
[--no-sidecars] [--overwrite] [--dry-run] [--report manifest.jsonl]
|
|
128
|
+
|
|
129
|
+
# Restore a compressed session to a working/scratch tree (bit-exact .dat) for
|
|
130
|
+
# SpikeInterface / Kilosort. Inverse of oe-session.
|
|
131
|
+
datapack oe-restore <compressed_session_dir> <work_dir> [--overwrite]
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Video
|
|
135
|
+
```sh
|
|
136
|
+
# Visually-lossless HEVC re-encode (NVENC; libx265 CPU fallback only with --allow-cpu).
|
|
137
|
+
datapack video <src.h264> <dst.mp4> [--quality safe|balanced|small] [--qp N] [--fps F] \
|
|
138
|
+
[--ffmpeg /path/to/ffmpeg] [--allow-cpu] [--report manifest.jsonl]
|
|
139
|
+
```
|
|
140
|
+
`--quality` presets (preset p7, GOP 100): `safe`=qp18, `balanced`=qp20 (default),
|
|
141
|
+
`small`=qp22. **Provisional** pending the lab's DLC/SLEAP tracking validation; `--qp`
|
|
142
|
+
overrides. Raw `.h264` has no timing, so `--fps` (default 100) is forced on input.
|
|
143
|
+
|
|
144
|
+
### Histology
|
|
145
|
+
```sh
|
|
146
|
+
# In-place lossless recompression of every TIFF in a tree (e.g. a brainreg
|
|
147
|
+
# registration/), copying non-TIFF (incl. brainreg .npy tracks) verbatim. Output
|
|
148
|
+
# files stay normal, natively-readable TIFFs (ADOBE_DEFLATE) — no unpack needed.
|
|
149
|
+
datapack histology tree <src_dir> <dst_dir> [--codec deflate|zstd] [--level N] \
|
|
150
|
+
[--no-tile] [--overwrite] [--dry-run] [--report manifest.jsonl]
|
|
151
|
+
|
|
152
|
+
# Optional bundling: recompress a TIFF dir then store it as ONE archive (adds no
|
|
153
|
+
# compression over `tree` — purely bundling for transfer/tape).
|
|
154
|
+
datapack histology pack <tiff_dir> <archive.zip> [--codec ...] [--level N] [--overwrite] [--dry-run]
|
|
155
|
+
datapack histology unpack <archive.zip> <work_dir> [--no-verify] # extract + re-verify
|
|
156
|
+
datapack histology repack <work_dir> <archive.zip> [--codec ...] [--level N] # after editing
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
### Datasets (dataset-aware "pack and stash")
|
|
160
|
+
```sh
|
|
161
|
+
# Pack a brainsaw/StitchIt serial_section/<sample>/ sample. Default: recompress every
|
|
162
|
+
# TIFF in place (raw + registration) -> stays browsable; copy .npy tracks + recipe/
|
|
163
|
+
# logs/mat/ini verbatim. Use --archive-raw to bundle stitchedImages/downsampled
|
|
164
|
+
# into one zip each (no ratio benefit, only one-object-per-dir for transfer/tape).
|
|
165
|
+
datapack serial-section <sample_dir> <dest_dir> [--archive-raw] [--codec deflate|zstd] \
|
|
166
|
+
[--overwrite] [--dry-run]
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Python API
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
import datapack
|
|
173
|
+
|
|
174
|
+
# one file (modality auto-detected or forced)
|
|
175
|
+
rec = datapack.compress_file("continuous.dat", "out.cbin",
|
|
176
|
+
params={"sample_rate": 30000, "n_channels": 384})
|
|
177
|
+
print(rec.ratio, rec.lossless, rec.verification.ok, rec.ready_to_offload)
|
|
178
|
+
|
|
179
|
+
# ephys session round-trip
|
|
180
|
+
datapack.compress_oe_session("session/", "compressed/") # -> .cbin/.ch + sidecars
|
|
181
|
+
datapack.decompress_oe_session("compressed/", "scratch/") # -> bit-exact .dat tree
|
|
182
|
+
|
|
183
|
+
# dataset-aware
|
|
184
|
+
datapack.pack_serial_section("serial_section/ap5l/", "packed/")
|
|
185
|
+
|
|
186
|
+
from datapack.engines import (
|
|
187
|
+
EphysEngine, HistologyEngine, VideoEngine, # Engine.compress(Job) -> ManifestRecord
|
|
188
|
+
compress_tiff_tree, pack, unpack, repack, # histology tree + archive
|
|
189
|
+
probe_ffmpeg, # video capability probe
|
|
190
|
+
)
|
|
191
|
+
from datapack.core import Job, ManifestRecord, ManifestWriter, read_manifest
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
Every engine returns a `ManifestRecord` (pydantic): source/output paths + sizes, ratio,
|
|
195
|
+
`lossless`, the `VerifyResult`, engine + tool versions, and engine-specific `extra` (e.g.
|
|
196
|
+
mtscomp `sha1_uncompressed`/`sha1_compressed`, or a TIFF decoded-pixel SHA-256). Write a
|
|
197
|
+
run's records to JSON-lines with `ManifestWriter` and read them back with `read_manifest`.
|
|
198
|
+
|
|
199
|
+
## Development setup
|
|
200
|
+
|
|
201
|
+
```sh
|
|
202
|
+
git clone https://github.com/larsrollik/datapack.git
|
|
203
|
+
cd datapack
|
|
204
|
+
uv sync --extra dev
|
|
205
|
+
uv run pre-commit install --hook-type pre-commit --hook-type commit-msg
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## Running tests
|
|
209
|
+
|
|
210
|
+
```sh
|
|
211
|
+
uv run pytest
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Tests needing optional deps or a GPU skip automatically when unavailable. To exercise the
|
|
215
|
+
real NVENC video path, point `$DATAPACK_FFMPEG` at an nvenc-enabled `ffmpeg`.
|
|
216
|
+
|
|
217
|
+
## Release workflow
|
|
218
|
+
|
|
219
|
+
1. Work on a `feature/` or `fix/` branch, committing with `cz commit`
|
|
220
|
+
2. Open a PR — CI (lint + tests + secrets scan) must pass before merge
|
|
221
|
+
3. Merge to main → version bump and tag are created automatically
|
|
222
|
+
4. Tag triggers release: GitHub release + PyPI publish
|
|
223
|
+
|
|
224
|
+
## License
|
|
225
|
+
|
|
226
|
+
See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# datapack
|
|
2
|
+
|
|
3
|
+
Lossless and visually-lossless compression of acquisition data (ephys, video,
|
|
4
|
+
histology) with verification and manifests.
|
|
5
|
+
|
|
6
|
+
`datapack` moves raw acquisition data into a compressed state, **verifies** every
|
|
7
|
+
artifact, and writes a **manifest** so a downstream offload/tape process can move it
|
|
8
|
+
and delete the original with confidence. It only ever **reads** the source — it never
|
|
9
|
+
mutates or deletes the raw data.
|
|
10
|
+
|
|
11
|
+
## What it handles
|
|
12
|
+
|
|
13
|
+
| Modality | Lossless? | Engine | Typical saving | Notes |
|
|
14
|
+
|---|---|---|---|---|
|
|
15
|
+
| **Ephys** (Neuropixels / Open Ephys) | **lossless** (bit-exact) | IBL `mtscomp` (`.cbin`+`.ch`) | ~⅓ (NP1 ~59%) | round-trips back to identical `.dat` |
|
|
16
|
+
| **Video** (behavioural, H.264) | **visually** lossless | `ffmpeg` HEVC (NVENC) | ~35–65% | re-encode for DLC/SLEAP; the `.mp4` is the working form |
|
|
17
|
+
| **Histology** (TIFF: brainsaw/StitchIt, brainreg) | **lossless** (bit-exact) | `tifffile` ADOBE_DEFLATE + predictor | ~⅔ (labels 40–75×) | files stay natively readable; tracks/metadata copied verbatim |
|
|
18
|
+
|
|
19
|
+
**Design principles**
|
|
20
|
+
|
|
21
|
+
- **Compress only the large data.** Metadata, `.npy` tracks, recipes, logs, JSON/CSV
|
|
22
|
+
are copied **byte-for-byte verbatim**.
|
|
23
|
+
- **Verify before trusting.** Lossless engines prove a bit-exact round-trip; video
|
|
24
|
+
checks frame-count + clean decode. Verification gates the artifact before it lands.
|
|
25
|
+
- **Round-trip in place.** Lossless outputs stay readable / restorable, so you can work
|
|
26
|
+
on them in scratch (spike-sorting, image processing) and recompress — no lock-in.
|
|
27
|
+
- **Never touch the source.** Write-to-temp → verify → atomic rename; originals untouched.
|
|
28
|
+
- **Manifests** record sizes, ratios, hashes, params and tool versions for the offload step.
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
pip install datapack-neuro # core (CLI + histology-independent pieces)
|
|
34
|
+
pip install "datapack-neuro[ephys]" # + mtscomp, numpy, tqdm (ephys engine)
|
|
35
|
+
pip install "datapack-neuro[histology]" # + tifffile, imagecodecs, numpy (histology engine)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The distribution is `datapack-neuro`; the import name is `datapack`. An unrelated
|
|
39
|
+
`datapack` project already holds that name on PyPI.
|
|
40
|
+
|
|
41
|
+
Video needs an **NVENC-enabled `ffmpeg`** on the system (not a Python dep). datapack
|
|
42
|
+
resolves it from `--ffmpeg`, then `$DATAPACK_FFMPEG`, then `PATH`. On a cluster, assume
|
|
43
|
+
the GPU is managed externally — datapack **fails loudly** if no working GPU encoder is
|
|
44
|
+
found rather than silently grinding on CPU (opt into the slow CPU path with `--allow-cpu`).
|
|
45
|
+
|
|
46
|
+
## CLI
|
|
47
|
+
|
|
48
|
+
Run `datapack <command> --help` for full flags. All commands take a source and a
|
|
49
|
+
destination and never modify the source.
|
|
50
|
+
|
|
51
|
+
### `compress` — one file, modality auto-detected
|
|
52
|
+
```sh
|
|
53
|
+
datapack compress <src> <dst> [--modality ephys|video|histology] [--dry-run] [--report manifest.jsonl]
|
|
54
|
+
```
|
|
55
|
+
Detects the modality from the path (`continuous.dat`→ephys, `.h264/.mp4`→video,
|
|
56
|
+
`.tif/.tiff`→histology) and runs the right engine. For tuned video/ephys runs prefer the
|
|
57
|
+
dedicated commands below.
|
|
58
|
+
|
|
59
|
+
### Ephys
|
|
60
|
+
```sh
|
|
61
|
+
# Compress a whole Open Ephys session: every continuous.dat -> .cbin/.ch,
|
|
62
|
+
# all sidecars (structure.oebin, *.npy, events/, settings.xml ...) copied verbatim.
|
|
63
|
+
datapack oe-session <session_dir> <dest_dir> [--threads N] [--chunk-duration S] \
|
|
64
|
+
[--no-sidecars] [--overwrite] [--dry-run] [--report manifest.jsonl]
|
|
65
|
+
|
|
66
|
+
# Restore a compressed session to a working/scratch tree (bit-exact .dat) for
|
|
67
|
+
# SpikeInterface / Kilosort. Inverse of oe-session.
|
|
68
|
+
datapack oe-restore <compressed_session_dir> <work_dir> [--overwrite]
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
### Video
|
|
72
|
+
```sh
|
|
73
|
+
# Visually-lossless HEVC re-encode (NVENC; libx265 CPU fallback only with --allow-cpu).
|
|
74
|
+
datapack video <src.h264> <dst.mp4> [--quality safe|balanced|small] [--qp N] [--fps F] \
|
|
75
|
+
[--ffmpeg /path/to/ffmpeg] [--allow-cpu] [--report manifest.jsonl]
|
|
76
|
+
```
|
|
77
|
+
`--quality` presets (preset p7, GOP 100): `safe`=qp18, `balanced`=qp20 (default),
|
|
78
|
+
`small`=qp22. **Provisional** pending the lab's DLC/SLEAP tracking validation; `--qp`
|
|
79
|
+
overrides. Raw `.h264` has no timing, so `--fps` (default 100) is forced on input.
|
|
80
|
+
|
|
81
|
+
### Histology
|
|
82
|
+
```sh
|
|
83
|
+
# In-place lossless recompression of every TIFF in a tree (e.g. a brainreg
|
|
84
|
+
# registration/), copying non-TIFF (incl. brainreg .npy tracks) verbatim. Output
|
|
85
|
+
# files stay normal, natively-readable TIFFs (ADOBE_DEFLATE) — no unpack needed.
|
|
86
|
+
datapack histology tree <src_dir> <dst_dir> [--codec deflate|zstd] [--level N] \
|
|
87
|
+
[--no-tile] [--overwrite] [--dry-run] [--report manifest.jsonl]
|
|
88
|
+
|
|
89
|
+
# Optional bundling: recompress a TIFF dir then store it as ONE archive (adds no
|
|
90
|
+
# compression over `tree` — purely bundling for transfer/tape).
|
|
91
|
+
datapack histology pack <tiff_dir> <archive.zip> [--codec ...] [--level N] [--overwrite] [--dry-run]
|
|
92
|
+
datapack histology unpack <archive.zip> <work_dir> [--no-verify] # extract + re-verify
|
|
93
|
+
datapack histology repack <work_dir> <archive.zip> [--codec ...] [--level N] # after editing
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Datasets (dataset-aware "pack and stash")
|
|
97
|
+
```sh
|
|
98
|
+
# Pack a brainsaw/StitchIt serial_section/<sample>/ sample. Default: recompress every
|
|
99
|
+
# TIFF in place (raw + registration) -> stays browsable; copy .npy tracks + recipe/
|
|
100
|
+
# logs/mat/ini verbatim. Use --archive-raw to bundle stitchedImages/downsampled
|
|
101
|
+
# into one zip each (no ratio benefit, only one-object-per-dir for transfer/tape).
|
|
102
|
+
datapack serial-section <sample_dir> <dest_dir> [--archive-raw] [--codec deflate|zstd] \
|
|
103
|
+
[--overwrite] [--dry-run]
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Python API
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import datapack
|
|
110
|
+
|
|
111
|
+
# one file (modality auto-detected or forced)
|
|
112
|
+
rec = datapack.compress_file("continuous.dat", "out.cbin",
|
|
113
|
+
params={"sample_rate": 30000, "n_channels": 384})
|
|
114
|
+
print(rec.ratio, rec.lossless, rec.verification.ok, rec.ready_to_offload)
|
|
115
|
+
|
|
116
|
+
# ephys session round-trip
|
|
117
|
+
datapack.compress_oe_session("session/", "compressed/") # -> .cbin/.ch + sidecars
|
|
118
|
+
datapack.decompress_oe_session("compressed/", "scratch/") # -> bit-exact .dat tree
|
|
119
|
+
|
|
120
|
+
# dataset-aware
|
|
121
|
+
datapack.pack_serial_section("serial_section/ap5l/", "packed/")
|
|
122
|
+
|
|
123
|
+
from datapack.engines import (
|
|
124
|
+
EphysEngine, HistologyEngine, VideoEngine, # Engine.compress(Job) -> ManifestRecord
|
|
125
|
+
compress_tiff_tree, pack, unpack, repack, # histology tree + archive
|
|
126
|
+
probe_ffmpeg, # video capability probe
|
|
127
|
+
)
|
|
128
|
+
from datapack.core import Job, ManifestRecord, ManifestWriter, read_manifest
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Every engine returns a `ManifestRecord` (pydantic): source/output paths + sizes, ratio,
|
|
132
|
+
`lossless`, the `VerifyResult`, engine + tool versions, and engine-specific `extra` (e.g.
|
|
133
|
+
mtscomp `sha1_uncompressed`/`sha1_compressed`, or a TIFF decoded-pixel SHA-256). Write a
|
|
134
|
+
run's records to JSON-lines with `ManifestWriter` and read them back with `read_manifest`.
|
|
135
|
+
|
|
136
|
+
## Development setup
|
|
137
|
+
|
|
138
|
+
```sh
|
|
139
|
+
git clone https://github.com/larsrollik/datapack.git
|
|
140
|
+
cd datapack
|
|
141
|
+
uv sync --extra dev
|
|
142
|
+
uv run pre-commit install --hook-type pre-commit --hook-type commit-msg
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Running tests
|
|
146
|
+
|
|
147
|
+
```sh
|
|
148
|
+
uv run pytest
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Tests needing optional deps or a GPU skip automatically when unavailable. To exercise the
|
|
152
|
+
real NVENC video path, point `$DATAPACK_FFMPEG` at an nvenc-enabled `ffmpeg`.
|
|
153
|
+
|
|
154
|
+
## Release workflow
|
|
155
|
+
|
|
156
|
+
1. Work on a `feature/` or `fix/` branch, committing with `cz commit`
|
|
157
|
+
2. Open a PR — CI (lint + tests + secrets scan) must pass before merge
|
|
158
|
+
3. Merge to main → version bump and tag are created automatically
|
|
159
|
+
4. Tag triggers release: GitHub release + PyPI publish
|
|
160
|
+
|
|
161
|
+
## License
|
|
162
|
+
|
|
163
|
+
See [LICENSE](LICENSE).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
1.2.0
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "datapack-neuro"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Lossless and visually-lossless compression of acquisition data (ephys, video, histology) with verification and manifests."
|
|
9
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
|
+
license = { file = "LICENSE" }
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Lars B. Rollik", email = "lars@rollik.me" },
|
|
13
|
+
]
|
|
14
|
+
requires-python = ">=3.12"
|
|
15
|
+
dependencies = [
|
|
16
|
+
"pydantic>=2", # manifest + config models
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
datapack = "datapack.cli:main"
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://github.com/larsrollik/datapack"
|
|
24
|
+
Documentation = "https://larsrollik.github.io/datapack/"
|
|
25
|
+
"Issue Tracker" = "https://github.com/larsrollik/datapack/issues"
|
|
26
|
+
|
|
27
|
+
# ---------------------------------------------------------------------------
|
|
28
|
+
# Build
|
|
29
|
+
# ---------------------------------------------------------------------------
|
|
30
|
+
|
|
31
|
+
[tool.hatch.version]
|
|
32
|
+
source = "vcs"
|
|
33
|
+
fallback-version = "1.0.0"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.hooks.vcs]
|
|
36
|
+
version-file = "src/datapack/_version.py"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/datapack"]
|
|
40
|
+
|
|
41
|
+
[tool.hatch.build.targets.sdist]
|
|
42
|
+
include = [
|
|
43
|
+
"/src/datapack",
|
|
44
|
+
"/tests",
|
|
45
|
+
"/README.md",
|
|
46
|
+
"/LICENSE",
|
|
47
|
+
"/pyproject.toml",
|
|
48
|
+
"/CITATION.cff",
|
|
49
|
+
"/VERSION",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
# Dependencies
|
|
54
|
+
# ---------------------------------------------------------------------------
|
|
55
|
+
|
|
56
|
+
[project.optional-dependencies]
|
|
57
|
+
ephys = [
|
|
58
|
+
"mtscomp==1.0.2", # IBL lossless ephys compression (.cbin/.ch); see ROADMAP D1/D6
|
|
59
|
+
"numpy>=2", # mtscomp runtime dep; numpy 2.x verified
|
|
60
|
+
"tqdm", # mtscomp imports it but its wheel does NOT declare it
|
|
61
|
+
]
|
|
62
|
+
histology = [
|
|
63
|
+
"tifffile>=2023", # lossless TIFF read/write (BigTIFF, tiled, predictor)
|
|
64
|
+
"imagecodecs", # codec backends (deflate, zstd) — manylinux wheels, no system libs
|
|
65
|
+
"numpy>=2", # pixel-array verification
|
|
66
|
+
]
|
|
67
|
+
dev = [
|
|
68
|
+
"commitizen",
|
|
69
|
+
"pytest>=8",
|
|
70
|
+
"pytest-cov",
|
|
71
|
+
"pre-commit",
|
|
72
|
+
"mypy",
|
|
73
|
+
"datapack[ephys]", # so the ephys engine is exercised in tests/CI
|
|
74
|
+
"datapack[histology]", # so the histology engine + archive are exercised in tests/CI
|
|
75
|
+
]
|
|
76
|
+
docs = [
|
|
77
|
+
"mkdocs-material",
|
|
78
|
+
# Uncomment to enable auto-generated API reference from docstrings:
|
|
79
|
+
# "mkdocstrings[python]",
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------------------
|
|
83
|
+
# Commitizen
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
[tool.commitizen]
|
|
87
|
+
name = "cz_conventional_commits"
|
|
88
|
+
version_provider = "commitizen"
|
|
89
|
+
version = "1.2.0"
|
|
90
|
+
version_files = ["VERSION"]
|
|
91
|
+
tag_format = "v$version"
|
|
92
|
+
update_changelog_on_bump = false
|
|
93
|
+
|
|
94
|
+
# ---------------------------------------------------------------------------
|
|
95
|
+
# Pytest
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
|
|
98
|
+
[tool.pytest.ini_options]
|
|
99
|
+
testpaths = ["tests"]
|
|
100
|
+
addopts = "--cov=src/datapack --cov-report=term-missing --maxfail=5 --import-mode=importlib"
|
|
101
|
+
|
|
102
|
+
# ---------------------------------------------------------------------------
|
|
103
|
+
# Ruff
|
|
104
|
+
# ---------------------------------------------------------------------------
|
|
105
|
+
|
|
106
|
+
[tool.ruff]
|
|
107
|
+
line-length = 88
|
|
108
|
+
indent-width = 4
|
|
109
|
+
src = ["src"]
|
|
110
|
+
|
|
111
|
+
[tool.ruff.lint]
|
|
112
|
+
select = [
|
|
113
|
+
"E", "W", # pycodestyle
|
|
114
|
+
"F", # pyflakes
|
|
115
|
+
"I", # isort
|
|
116
|
+
"UP", # pyupgrade
|
|
117
|
+
"B", # flake8-bugbear
|
|
118
|
+
"SIM", # flake8-simplify
|
|
119
|
+
"PTH", # flake8-pathlib
|
|
120
|
+
"TCH", # flake8-type-checking
|
|
121
|
+
"PYI", # flake8-pyi
|
|
122
|
+
"YTT", # flake8-2020
|
|
123
|
+
"N", # pep8-naming
|
|
124
|
+
]
|
|
125
|
+
ignore = [
|
|
126
|
+
"N818", # keep expressive exception names (EngineNotAvailable) without an Error suffix
|
|
127
|
+
]
|
|
128
|
+
fixable = ["ALL"]
|
|
129
|
+
|
|
130
|
+
[tool.ruff.format]
|
|
131
|
+
quote-style = "double"
|
|
132
|
+
indent-style = "space"
|
|
133
|
+
docstring-code-format = true
|
|
134
|
+
|
|
135
|
+
[tool.ruff.lint.pydocstyle]
|
|
136
|
+
convention = "numpy"
|
|
137
|
+
|
|
138
|
+
[tool.ruff.lint.flake8-type-checking]
|
|
139
|
+
# pydantic models evaluate annotations at runtime, so field-type imports must
|
|
140
|
+
# stay at module scope — don't let TC rules push them into TYPE_CHECKING blocks.
|
|
141
|
+
runtime-evaluated-base-classes = ["pydantic.BaseModel"]
|
|
142
|
+
|
|
143
|
+
[tool.ruff.lint.per-file-ignores]
|
|
144
|
+
"tests/**" = ["TC001", "TC002", "TC003"] # annotation-only imports are fine in tests
|
|
145
|
+
|
|
146
|
+
# ---------------------------------------------------------------------------
|
|
147
|
+
# Mypy
|
|
148
|
+
# ---------------------------------------------------------------------------
|
|
149
|
+
|
|
150
|
+
[tool.mypy]
|
|
151
|
+
python_version = "3.12"
|
|
152
|
+
warn_return_any = true
|
|
153
|
+
warn_unused_ignores = true
|
|
154
|
+
ignore_missing_imports = true
|