datapack-neuro 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- datapack/__init__.py +124 -0
- datapack/_version.py +24 -0
- datapack/cli.py +379 -0
- datapack/config.py +62 -0
- datapack/core/__init__.py +41 -0
- datapack/core/atomic.py +90 -0
- datapack/core/detect.py +55 -0
- datapack/core/engine.py +39 -0
- datapack/core/errors.py +40 -0
- datapack/core/job.py +37 -0
- datapack/core/manifest.py +83 -0
- datapack/core/verify.py +45 -0
- datapack/datasets.py +143 -0
- datapack/engines/__init__.py +53 -0
- datapack/engines/ephys.py +595 -0
- datapack/engines/histology.py +584 -0
- datapack/engines/video.py +433 -0
- datapack/py.typed +0 -0
- datapack_neuro-1.2.0.dist-info/METADATA +226 -0
- datapack_neuro-1.2.0.dist-info/RECORD +23 -0
- datapack_neuro-1.2.0.dist-info/WHEEL +4 -0
- datapack_neuro-1.2.0.dist-info/entry_points.txt +2 -0
- datapack_neuro-1.2.0.dist-info/licenses/LICENSE +32 -0
datapack/__init__.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""datapack — verified compression of acquisition data (ephys, video, histology).
|
|
2
|
+
|
|
3
|
+
Public API (stable; this is what downstream pipelines pin):
|
|
4
|
+
|
|
5
|
+
- :func:`compress_file` — detect modality, run the engine, return a manifest record.
|
|
6
|
+
- :class:`~datapack.core.manifest.ManifestRecord` (aliased ``CompressResult``).
|
|
7
|
+
- :class:`~datapack.core.job.Job`, the core types, and :data:`config.DEFAULT_CONFIG`.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
# Import engines for their registration side effect (so get_engine_class works
|
|
17
|
+
# after `import datapack`). Engine modules import heavy deps lazily, so this is
|
|
18
|
+
# cheap and safe even when e.g. mtscomp is not installed.
|
|
19
|
+
from datapack import engines as _engines # noqa: E402,F401
|
|
20
|
+
from datapack.config import DEFAULT_CONFIG, Config
|
|
21
|
+
from datapack.core import (
|
|
22
|
+
DatapackError,
|
|
23
|
+
Engine,
|
|
24
|
+
EngineNotAvailable,
|
|
25
|
+
Job,
|
|
26
|
+
ManifestRecord,
|
|
27
|
+
ManifestWriter,
|
|
28
|
+
ModalityDetectionError,
|
|
29
|
+
VerificationError,
|
|
30
|
+
VerifyResult,
|
|
31
|
+
available_modalities,
|
|
32
|
+
detect_modality,
|
|
33
|
+
get_engine_class,
|
|
34
|
+
read_manifest,
|
|
35
|
+
register_engine,
|
|
36
|
+
sha256_file,
|
|
37
|
+
)
|
|
38
|
+
from datapack.datasets import SerialSectionResult, pack_serial_section # noqa: E402
|
|
39
|
+
from datapack.engines import ( # noqa: E402
|
|
40
|
+
compress_oe_session,
|
|
41
|
+
decompress_oe_session,
|
|
42
|
+
)
|
|
43
|
+
from datapack.engines.ephys import ( # noqa: E402
|
|
44
|
+
StreamMeta,
|
|
45
|
+
oebin_stream_meta,
|
|
46
|
+
session_stream_meta,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
try:
|
|
50
|
+
__version__ = version("datapack")
|
|
51
|
+
except PackageNotFoundError: # pragma: no cover
|
|
52
|
+
__version__ = "unknown"
|
|
53
|
+
|
|
54
|
+
# Name used in the design docs for the result of a compress operation.
|
|
55
|
+
CompressResult = ManifestRecord
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def compress_file(
|
|
59
|
+
src: Path | str,
|
|
60
|
+
dst: Path | str,
|
|
61
|
+
*,
|
|
62
|
+
modality: str | None = None,
|
|
63
|
+
params: dict[str, Any] | None = None,
|
|
64
|
+
) -> ManifestRecord:
|
|
65
|
+
"""Compress one file, returning its manifest record.
|
|
66
|
+
|
|
67
|
+
Parameters
|
|
68
|
+
----------
|
|
69
|
+
src
|
|
70
|
+
Source path (never modified).
|
|
71
|
+
dst
|
|
72
|
+
Artifact destination (must differ from ``src``).
|
|
73
|
+
modality
|
|
74
|
+
Force a modality; if None, it is detected from ``src``.
|
|
75
|
+
params
|
|
76
|
+
Engine-specific parameter overrides.
|
|
77
|
+
|
|
78
|
+
Raises
|
|
79
|
+
------
|
|
80
|
+
ModalityDetectionError
|
|
81
|
+
If ``modality`` is None and the source cannot be classified.
|
|
82
|
+
EngineNotAvailable
|
|
83
|
+
If the modality has no registered engine yet (e.g. during early phases).
|
|
84
|
+
"""
|
|
85
|
+
src = Path(src)
|
|
86
|
+
modality = modality or detect_modality(src)
|
|
87
|
+
if modality is None:
|
|
88
|
+
raise ModalityDetectionError(
|
|
89
|
+
f"could not detect modality for {src}; pass modality= explicitly"
|
|
90
|
+
)
|
|
91
|
+
engine = get_engine_class(modality)()
|
|
92
|
+
job = Job(src=src, dst=Path(dst), modality=modality, params=params or {})
|
|
93
|
+
return engine.compress(job)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
__all__ = [
|
|
97
|
+
"StreamMeta",
|
|
98
|
+
"oebin_stream_meta",
|
|
99
|
+
"session_stream_meta",
|
|
100
|
+
"__version__",
|
|
101
|
+
"compress_file",
|
|
102
|
+
"compress_oe_session",
|
|
103
|
+
"decompress_oe_session",
|
|
104
|
+
"pack_serial_section",
|
|
105
|
+
"SerialSectionResult",
|
|
106
|
+
"CompressResult",
|
|
107
|
+
"Config",
|
|
108
|
+
"DEFAULT_CONFIG",
|
|
109
|
+
"DatapackError",
|
|
110
|
+
"EngineNotAvailable",
|
|
111
|
+
"Engine",
|
|
112
|
+
"Job",
|
|
113
|
+
"ManifestRecord",
|
|
114
|
+
"ManifestWriter",
|
|
115
|
+
"ModalityDetectionError",
|
|
116
|
+
"VerificationError",
|
|
117
|
+
"VerifyResult",
|
|
118
|
+
"available_modalities",
|
|
119
|
+
"detect_modality",
|
|
120
|
+
"get_engine_class",
|
|
121
|
+
"read_manifest",
|
|
122
|
+
"register_engine",
|
|
123
|
+
"sha256_file",
|
|
124
|
+
]
|
datapack/_version.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '1.2.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 2, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
datapack/cli.py
ADDED
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""Command-line interface: ``datapack compress <path>``.
|
|
2
|
+
|
|
3
|
+
Phase 0 wires the pipeline end-to-end — detect modality, build the job, hand off
|
|
4
|
+
to the engine, and append the manifest record. ``--dry-run`` reports the plan
|
|
5
|
+
without touching anything; with no engines registered yet, a real run fails
|
|
6
|
+
loudly with ``EngineNotAvailable`` (it never silently skips compression).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from datapack import (
|
|
16
|
+
__version__,
|
|
17
|
+
compress_file,
|
|
18
|
+
compress_oe_session,
|
|
19
|
+
decompress_oe_session,
|
|
20
|
+
pack_serial_section,
|
|
21
|
+
)
|
|
22
|
+
from datapack.core import (
|
|
23
|
+
DatapackError,
|
|
24
|
+
ManifestWriter,
|
|
25
|
+
available_modalities,
|
|
26
|
+
detect_modality,
|
|
27
|
+
)
|
|
28
|
+
from datapack.engines import compress_tiff_tree, pack, repack, unpack
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
32
|
+
parser = argparse.ArgumentParser(
|
|
33
|
+
prog="datapack",
|
|
34
|
+
description="Verified compression of acquisition data.",
|
|
35
|
+
)
|
|
36
|
+
parser.add_argument(
|
|
37
|
+
"--version", action="version", version=f"datapack {__version__}"
|
|
38
|
+
)
|
|
39
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
40
|
+
|
|
41
|
+
c = sub.add_parser("compress", help="compress one source file")
|
|
42
|
+
c.add_argument("source", type=Path, help="source file to compress")
|
|
43
|
+
c.add_argument("dest", type=Path, help="artifact destination path")
|
|
44
|
+
c.add_argument(
|
|
45
|
+
"--modality",
|
|
46
|
+
choices=["ephys", "video", "histology"],
|
|
47
|
+
help="force modality (default: auto-detect from the source)",
|
|
48
|
+
)
|
|
49
|
+
c.add_argument(
|
|
50
|
+
"--dry-run",
|
|
51
|
+
action="store_true",
|
|
52
|
+
help="report the plan (detected modality, target) without compressing",
|
|
53
|
+
)
|
|
54
|
+
c.add_argument(
|
|
55
|
+
"--report",
|
|
56
|
+
type=Path,
|
|
57
|
+
help="append the manifest record to this JSON-lines file",
|
|
58
|
+
)
|
|
59
|
+
c.set_defaults(func=_cmd_compress)
|
|
60
|
+
|
|
61
|
+
s = sub.add_parser(
|
|
62
|
+
"oe-session",
|
|
63
|
+
help="compress every continuous.dat in an OpenEphys session tree",
|
|
64
|
+
)
|
|
65
|
+
s.add_argument("source", type=Path, help="OpenEphys session directory")
|
|
66
|
+
s.add_argument("dest", type=Path, help="destination directory (tree mirrored)")
|
|
67
|
+
s.add_argument("--threads", type=int, default=8)
|
|
68
|
+
s.add_argument("--chunk-duration", type=float, default=1.0, dest="chunk_duration")
|
|
69
|
+
s.add_argument(
|
|
70
|
+
"--no-sidecars",
|
|
71
|
+
action="store_true",
|
|
72
|
+
help="do not copy non-continuous.dat files into the destination",
|
|
73
|
+
)
|
|
74
|
+
s.add_argument("--overwrite", action="store_true")
|
|
75
|
+
s.add_argument("--dry-run", action="store_true", help="walk + resolve meta only")
|
|
76
|
+
s.add_argument(
|
|
77
|
+
"--report",
|
|
78
|
+
type=Path,
|
|
79
|
+
help="write per-stream manifest records here (JSON-lines)",
|
|
80
|
+
)
|
|
81
|
+
s.set_defaults(func=_cmd_oe_session)
|
|
82
|
+
|
|
83
|
+
r = sub.add_parser(
|
|
84
|
+
"oe-restore",
|
|
85
|
+
help="decompress a compressed OE session back to a working tree (round-trip)",
|
|
86
|
+
)
|
|
87
|
+
r.add_argument("source", type=Path, help="compressed OE session directory")
|
|
88
|
+
r.add_argument("dest", type=Path, help="working/scratch dir to restore into")
|
|
89
|
+
r.add_argument("--overwrite", action="store_true")
|
|
90
|
+
r.set_defaults(func=_cmd_oe_restore)
|
|
91
|
+
|
|
92
|
+
v = sub.add_parser("video", help="visually-lossless HEVC re-encode (NVENC)")
|
|
93
|
+
v.add_argument("source", type=Path, help="video file (e.g. .h264)")
|
|
94
|
+
v.add_argument("dest", type=Path, help="output .mp4")
|
|
95
|
+
v.add_argument(
|
|
96
|
+
"--qp", type=int, default=20, help="constqp value (18 safe..22 small)"
|
|
97
|
+
)
|
|
98
|
+
v.add_argument(
|
|
99
|
+
"--quality",
|
|
100
|
+
choices=["safe", "balanced", "small"],
|
|
101
|
+
help="named QP preset (safe=18, balanced=20, small=22); overrides --qp. "
|
|
102
|
+
"PROVISIONAL pending DLC validation",
|
|
103
|
+
)
|
|
104
|
+
v.add_argument("--fps", type=float, default=100.0, help="forced fps for raw .h264")
|
|
105
|
+
v.add_argument(
|
|
106
|
+
"--ffmpeg", help="path to an nvenc-enabled ffmpeg (or $DATAPACK_FFMPEG)"
|
|
107
|
+
)
|
|
108
|
+
v.add_argument(
|
|
109
|
+
"--allow-cpu",
|
|
110
|
+
action="store_true",
|
|
111
|
+
help="permit the slow libx265 CPU fallback if no GPU (emits a warning)",
|
|
112
|
+
)
|
|
113
|
+
v.add_argument("--report", type=Path, help="append manifest record (JSON-lines)")
|
|
114
|
+
v.set_defaults(func=_cmd_video)
|
|
115
|
+
|
|
116
|
+
ss = sub.add_parser(
|
|
117
|
+
"serial-section",
|
|
118
|
+
help="pack a brainsaw/StitchIt serial_section sample (dataset-aware)",
|
|
119
|
+
)
|
|
120
|
+
ss.add_argument("source", type=Path, help="serial_section/<sample>/ directory")
|
|
121
|
+
ss.add_argument("dest", type=Path, help="destination directory")
|
|
122
|
+
ss.add_argument(
|
|
123
|
+
"--archive-raw",
|
|
124
|
+
action="store_true",
|
|
125
|
+
help="bundle stitchedImages/downsampled into one zip each (default: recompress "
|
|
126
|
+
"in place, browsable). Zipping adds no ratio — only one-object-per-dir "
|
|
127
|
+
"bundling.",
|
|
128
|
+
)
|
|
129
|
+
ss.add_argument("--codec", choices=["deflate", "zstd"], default="deflate")
|
|
130
|
+
ss.add_argument("--overwrite", action="store_true")
|
|
131
|
+
ss.add_argument("--dry-run", action="store_true")
|
|
132
|
+
ss.set_defaults(func=_cmd_serial_section)
|
|
133
|
+
|
|
134
|
+
h = sub.add_parser("histology", help="TIFF archive workflow (pack/unpack/repack)")
|
|
135
|
+
hsub = h.add_subparsers(dest="hcommand", required=True)
|
|
136
|
+
|
|
137
|
+
hp = hsub.add_parser("pack", help="bundle a TIFF dir into one archive")
|
|
138
|
+
hp.add_argument(
|
|
139
|
+
"source", type=Path, help="TIFF directory (e.g. stitchedImages_100)"
|
|
140
|
+
)
|
|
141
|
+
hp.add_argument("archive", type=Path, help="output .zip archive")
|
|
142
|
+
hp.add_argument("--codec", choices=["deflate", "zstd"], default="deflate")
|
|
143
|
+
hp.add_argument("--level", type=int, default=6)
|
|
144
|
+
hp.add_argument("--overwrite", action="store_true")
|
|
145
|
+
hp.add_argument("--dry-run", action="store_true")
|
|
146
|
+
hp.set_defaults(func=_cmd_hist_pack)
|
|
147
|
+
|
|
148
|
+
hu = hsub.add_parser("unpack", help="extract an archive to a working dir + verify")
|
|
149
|
+
hu.add_argument("archive", type=Path)
|
|
150
|
+
hu.add_argument("dest", type=Path, help="working directory")
|
|
151
|
+
hu.add_argument("--no-verify", action="store_true")
|
|
152
|
+
hu.set_defaults(func=_cmd_hist_unpack)
|
|
153
|
+
|
|
154
|
+
hr = hsub.add_parser("repack", help="re-bundle a working dir after editing")
|
|
155
|
+
hr.add_argument("workdir", type=Path)
|
|
156
|
+
hr.add_argument("archive", type=Path)
|
|
157
|
+
hr.add_argument("--codec", choices=["deflate", "zstd"], default="deflate")
|
|
158
|
+
hr.add_argument("--level", type=int, default=6)
|
|
159
|
+
hr.set_defaults(func=_cmd_hist_repack)
|
|
160
|
+
|
|
161
|
+
ht = hsub.add_parser(
|
|
162
|
+
"tree",
|
|
163
|
+
help="recompress every TIFF in a tree in place (Tier 2; stays browsable)",
|
|
164
|
+
)
|
|
165
|
+
ht.add_argument(
|
|
166
|
+
"source", type=Path, help="dir tree (e.g. a brainreg registration/)"
|
|
167
|
+
)
|
|
168
|
+
ht.add_argument("dest", type=Path, help="destination tree (structure mirrored)")
|
|
169
|
+
ht.add_argument("--codec", choices=["deflate", "zstd"], default="deflate")
|
|
170
|
+
ht.add_argument("--level", type=int, default=6)
|
|
171
|
+
ht.add_argument(
|
|
172
|
+
"--no-tile",
|
|
173
|
+
action="store_true",
|
|
174
|
+
help="write stripped (not tiled) TIFFs for maximal legacy-reader compatibility",
|
|
175
|
+
)
|
|
176
|
+
ht.add_argument("--overwrite", action="store_true")
|
|
177
|
+
ht.add_argument("--dry-run", action="store_true")
|
|
178
|
+
ht.add_argument("--report", type=Path, help="per-file manifest (JSON-lines)")
|
|
179
|
+
ht.set_defaults(func=_cmd_hist_tree)
|
|
180
|
+
return parser
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _cmd_compress(args: argparse.Namespace) -> int:
|
|
184
|
+
modality = args.modality or detect_modality(args.source)
|
|
185
|
+
if modality is None:
|
|
186
|
+
print(
|
|
187
|
+
f"error: could not detect modality for {args.source}; pass --modality",
|
|
188
|
+
file=sys.stderr,
|
|
189
|
+
)
|
|
190
|
+
return 2
|
|
191
|
+
|
|
192
|
+
if args.dry_run:
|
|
193
|
+
engines = available_modalities()
|
|
194
|
+
engine_state = "engine ready" if modality in engines else "no engine yet"
|
|
195
|
+
print(f"[dry-run] {args.source} -> {args.dest}")
|
|
196
|
+
print(f" modality={modality} ({engine_state})")
|
|
197
|
+
return 0
|
|
198
|
+
|
|
199
|
+
record = compress_file(args.source, args.dest, modality=modality)
|
|
200
|
+
if args.report:
|
|
201
|
+
ManifestWriter(args.report).append(record)
|
|
202
|
+
saved = record.saved_fraction
|
|
203
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
204
|
+
print(f"ok: {args.source} -> {record.output_path} ({saved_str})")
|
|
205
|
+
return 0
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _cmd_oe_session(args: argparse.Namespace) -> int:
|
|
209
|
+
result = compress_oe_session(
|
|
210
|
+
args.source,
|
|
211
|
+
args.dest,
|
|
212
|
+
n_threads=args.threads,
|
|
213
|
+
chunk_duration_s=args.chunk_duration,
|
|
214
|
+
copy_sidecars=not args.no_sidecars,
|
|
215
|
+
overwrite=args.overwrite,
|
|
216
|
+
dry_run=args.dry_run,
|
|
217
|
+
manifest_path=args.report,
|
|
218
|
+
)
|
|
219
|
+
if args.dry_run:
|
|
220
|
+
n_ok = result.n_streams - len(result.skipped)
|
|
221
|
+
print(
|
|
222
|
+
f"[dry-run] {args.source}: {result.n_streams} continuous.dat stream(s) "
|
|
223
|
+
f"({n_ok} compressible, {len(result.skipped)} without meta)"
|
|
224
|
+
)
|
|
225
|
+
else:
|
|
226
|
+
saved = result.saved_fraction
|
|
227
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
228
|
+
print(
|
|
229
|
+
f"ok: {len(result.records)} stream(s) compressed ({saved_str}), "
|
|
230
|
+
f"{result.sidecars_copied} sidecar(s) copied -> {result.dest}"
|
|
231
|
+
)
|
|
232
|
+
if result.skipped:
|
|
233
|
+
print(f"warning: skipped {len(result.skipped)} stream(s) with no oebin meta:")
|
|
234
|
+
for rel in result.skipped:
|
|
235
|
+
print(f" - {rel}")
|
|
236
|
+
return 0
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _cmd_oe_restore(args: argparse.Namespace) -> int:
|
|
240
|
+
result = decompress_oe_session(args.source, args.dest, overwrite=args.overwrite)
|
|
241
|
+
print(
|
|
242
|
+
f"ok: restored {result.n_streams} stream(s) + "
|
|
243
|
+
f"{result.sidecars_copied} sidecar(s) -> {result.dest}"
|
|
244
|
+
)
|
|
245
|
+
return 0
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _cmd_video(args: argparse.Namespace) -> int:
|
|
249
|
+
record = compress_file(
|
|
250
|
+
args.source,
|
|
251
|
+
args.dest,
|
|
252
|
+
modality="video",
|
|
253
|
+
params={
|
|
254
|
+
"qp": args.qp,
|
|
255
|
+
"quality": args.quality,
|
|
256
|
+
"fps": args.fps,
|
|
257
|
+
"ffmpeg": args.ffmpeg,
|
|
258
|
+
"allow_cpu": args.allow_cpu,
|
|
259
|
+
},
|
|
260
|
+
)
|
|
261
|
+
if args.report:
|
|
262
|
+
ManifestWriter(args.report).append(record)
|
|
263
|
+
saved = record.saved_fraction
|
|
264
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
265
|
+
v = record.verification
|
|
266
|
+
fc = v.details.get("frame_count_match") if v else None
|
|
267
|
+
print(
|
|
268
|
+
f"ok: {args.source} -> {record.output_path} ({saved_str}; "
|
|
269
|
+
f"encoder={record.extra.get('encoder')}, frames_match={fc})"
|
|
270
|
+
)
|
|
271
|
+
return 0
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _cmd_hist_pack(args: argparse.Namespace) -> int:
|
|
275
|
+
result = pack(
|
|
276
|
+
args.source,
|
|
277
|
+
args.archive,
|
|
278
|
+
codec=args.codec,
|
|
279
|
+
level=args.level,
|
|
280
|
+
overwrite=args.overwrite,
|
|
281
|
+
dry_run=args.dry_run,
|
|
282
|
+
)
|
|
283
|
+
if args.dry_run:
|
|
284
|
+
print(
|
|
285
|
+
f"[dry-run] {args.source}: {result.n_tiffs} TIFF(s) + "
|
|
286
|
+
f"{result.n_other} other file(s) -> {args.archive}"
|
|
287
|
+
)
|
|
288
|
+
return 0
|
|
289
|
+
saved = result.saved_fraction
|
|
290
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
291
|
+
print(
|
|
292
|
+
f"ok: packed {result.n_tiffs} TIFF(s) + {result.n_other} other -> "
|
|
293
|
+
f"{result.archive} ({saved_str})"
|
|
294
|
+
)
|
|
295
|
+
return 0
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _cmd_hist_unpack(args: argparse.Namespace) -> int:
|
|
299
|
+
result = unpack(args.archive, args.dest, verify=not args.no_verify)
|
|
300
|
+
verified = "verified" if result.verified else "NOT verified"
|
|
301
|
+
print(f"ok: unpacked {result.n_files} file(s) -> {result.dest} ({verified})")
|
|
302
|
+
return 0
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _cmd_hist_repack(args: argparse.Namespace) -> int:
|
|
306
|
+
result = repack(args.workdir, args.archive, codec=args.codec, level=args.level)
|
|
307
|
+
saved = result.saved_fraction
|
|
308
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
309
|
+
print(
|
|
310
|
+
f"ok: repacked {result.n_tiffs} TIFF(s) + {result.n_other} other -> "
|
|
311
|
+
f"{result.archive} ({saved_str})"
|
|
312
|
+
)
|
|
313
|
+
return 0
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _cmd_serial_section(args: argparse.Namespace) -> int:
|
|
317
|
+
result = pack_serial_section(
|
|
318
|
+
args.source,
|
|
319
|
+
args.dest,
|
|
320
|
+
archive_raw=args.archive_raw,
|
|
321
|
+
codec=args.codec,
|
|
322
|
+
overwrite=args.overwrite,
|
|
323
|
+
dry_run=args.dry_run,
|
|
324
|
+
)
|
|
325
|
+
if args.dry_run:
|
|
326
|
+
print(
|
|
327
|
+
f"[dry-run] {args.source}: "
|
|
328
|
+
f"{len(result.archived_dirs)} dir(s) to archive, "
|
|
329
|
+
f"{len(result.recompressed_dirs)} dir(s) to recompress, "
|
|
330
|
+
f"{result.files_copied} top-level file(s) to copy"
|
|
331
|
+
)
|
|
332
|
+
return 0
|
|
333
|
+
saved = result.saved_fraction
|
|
334
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
335
|
+
print(
|
|
336
|
+
f"ok: {len(result.archived_dirs)} archive(s), "
|
|
337
|
+
f"{len(result.recompressed_dirs)} recompressed dir(s), "
|
|
338
|
+
f"{result.files_copied} file(s) copied -> {result.dest} ({saved_str})"
|
|
339
|
+
)
|
|
340
|
+
return 0
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _cmd_hist_tree(args: argparse.Namespace) -> int:
|
|
344
|
+
result = compress_tiff_tree(
|
|
345
|
+
args.source,
|
|
346
|
+
args.dest,
|
|
347
|
+
codec=args.codec,
|
|
348
|
+
level=args.level,
|
|
349
|
+
tile=None if args.no_tile else (256, 256),
|
|
350
|
+
overwrite=args.overwrite,
|
|
351
|
+
dry_run=args.dry_run,
|
|
352
|
+
manifest_path=args.report,
|
|
353
|
+
)
|
|
354
|
+
if args.dry_run:
|
|
355
|
+
print(
|
|
356
|
+
f"[dry-run] {args.source}: {result.n_tiffs} TIFF(s) would be recompressed"
|
|
357
|
+
)
|
|
358
|
+
return 0
|
|
359
|
+
saved = result.saved_fraction
|
|
360
|
+
saved_str = f"{saved:.1%} saved" if saved is not None else "ratio n/a"
|
|
361
|
+
print(
|
|
362
|
+
f"ok: recompressed {len(result.records)} TIFF(s) + "
|
|
363
|
+
f"{result.sidecars_copied} sidecar(s) -> {result.dest} ({saved_str})"
|
|
364
|
+
)
|
|
365
|
+
return 0
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def main(argv: list[str] | None = None) -> int:
|
|
369
|
+
parser = _build_parser()
|
|
370
|
+
args = parser.parse_args(argv)
|
|
371
|
+
try:
|
|
372
|
+
return int(args.func(args))
|
|
373
|
+
except DatapackError as exc:
|
|
374
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
375
|
+
return 1
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
if __name__ == "__main__": # pragma: no cover
|
|
379
|
+
raise SystemExit(main())
|
datapack/config.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Default compression profiles per modality.
|
|
2
|
+
|
|
3
|
+
These are *data*: the values the engines will consume in later phases. They
|
|
4
|
+
capture the project's compression decisions so the defaults
|
|
5
|
+
are reviewable in one place. Engines are not implemented yet (Phase 1-3).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EphysProfile(BaseModel):
|
|
14
|
+
"""IBL mtscomp, truly lossless."""
|
|
15
|
+
|
|
16
|
+
codec: str = "mtscomp"
|
|
17
|
+
mtscomp_pin: str = "==1.0.2"
|
|
18
|
+
# NOTE: an earlier pipeline measured 120 s chunks; mtscomp's API default is 1 s.
|
|
19
|
+
# Resolve this gap explicitly in Phase 1 (affects ratio + random access).
|
|
20
|
+
chunk_duration_s: float = 1.0
|
|
21
|
+
n_threads: int = 8
|
|
22
|
+
# mtscomp's built-in decompress+byte-compare IS the integrity gate (D6).
|
|
23
|
+
check_after_compress: bool = True
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class VideoProfile(BaseModel):
|
|
27
|
+
"""ffmpeg HEVC, visually lossless."""
|
|
28
|
+
|
|
29
|
+
encoder: str = "hevc_nvenc"
|
|
30
|
+
qp: int = 20 # qp18 = max-safe, qp22 = max-shrink; lock via DLC/SLEAP validation
|
|
31
|
+
# Named presets from the jimbob DLC validation set (preset p7, g100).
|
|
32
|
+
# PROVISIONAL — tracking-accuracy validation not finished.
|
|
33
|
+
qp_presets: dict[str, int] = {"safe": 18, "balanced": 20, "small": 22}
|
|
34
|
+
preset: str = "p7"
|
|
35
|
+
gop: int = 100
|
|
36
|
+
bframes: int = 0
|
|
37
|
+
spatial_aq: bool = True
|
|
38
|
+
pix_fmt: str = "yuv420p"
|
|
39
|
+
container: str = "mp4"
|
|
40
|
+
fallback_encoder: str = "libx265"
|
|
41
|
+
fallback_crf: int = 20
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class HistologyProfile(BaseModel):
|
|
45
|
+
"""Lossless tiled BigTIFF."""
|
|
46
|
+
|
|
47
|
+
codec: str = "deflate" # promote to "zstd" after lab-reader compat check
|
|
48
|
+
level: int = 6
|
|
49
|
+
predictor: bool = True
|
|
50
|
+
tile: tuple[int, int] = (256, 256)
|
|
51
|
+
bigtiff: bool = True
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class Config(BaseModel):
|
|
55
|
+
"""Top-level config bundling the per-modality profiles."""
|
|
56
|
+
|
|
57
|
+
ephys: EphysProfile = Field(default_factory=EphysProfile)
|
|
58
|
+
video: VideoProfile = Field(default_factory=VideoProfile)
|
|
59
|
+
histology: HistologyProfile = Field(default_factory=HistologyProfile)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
DEFAULT_CONFIG = Config()
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""datapack core: the modality-independent compress/verify/manifest contract."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datapack.core.atomic import atomic_write, atomic_write_many
|
|
6
|
+
from datapack.core.detect import (
|
|
7
|
+
available_modalities,
|
|
8
|
+
detect_modality,
|
|
9
|
+
get_engine_class,
|
|
10
|
+
register_engine,
|
|
11
|
+
)
|
|
12
|
+
from datapack.core.engine import Engine
|
|
13
|
+
from datapack.core.errors import (
|
|
14
|
+
DatapackError,
|
|
15
|
+
EngineNotAvailable,
|
|
16
|
+
ModalityDetectionError,
|
|
17
|
+
VerificationError,
|
|
18
|
+
)
|
|
19
|
+
from datapack.core.job import Job
|
|
20
|
+
from datapack.core.manifest import ManifestRecord, ManifestWriter, read_manifest
|
|
21
|
+
from datapack.core.verify import VerifyResult, sha256_file
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"atomic_write",
|
|
25
|
+
"atomic_write_many",
|
|
26
|
+
"available_modalities",
|
|
27
|
+
"detect_modality",
|
|
28
|
+
"get_engine_class",
|
|
29
|
+
"register_engine",
|
|
30
|
+
"Engine",
|
|
31
|
+
"DatapackError",
|
|
32
|
+
"EngineNotAvailable",
|
|
33
|
+
"ModalityDetectionError",
|
|
34
|
+
"VerificationError",
|
|
35
|
+
"Job",
|
|
36
|
+
"ManifestRecord",
|
|
37
|
+
"ManifestWriter",
|
|
38
|
+
"read_manifest",
|
|
39
|
+
"VerifyResult",
|
|
40
|
+
"sha256_file",
|
|
41
|
+
]
|