dsimaging-admin 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/PKG-INFO +1 -1
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/pyproject.toml +1 -1
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/__init__.py +1 -1
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/manifest.py +29 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/store.py +3 -1
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/.gitignore +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/IMPROVEMENTS.md +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/LICENSE +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/README.md +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/cli.py +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/controller.py +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/hashing.py +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/s3.py +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/src/dsimaging_admin/verify.py +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/tests/test_admin_features.py +0 -0
- {dsimaging_admin-0.3.0 → dsimaging_admin-0.3.2}/tests/test_manifest.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dsimaging-admin
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Admin CLI for managing medical imaging datasets in S3/MinIO for DataSHIELD
|
|
5
5
|
Project-URL: Homepage, https://github.com/isglobal-brge/dsimaging-admin
|
|
6
6
|
Project-URL: Repository, https://github.com/isglobal-brge/dsimaging-admin
|
|
@@ -321,6 +321,17 @@ def write_manifest_yaml(manifest: dict, path: str):
|
|
|
321
321
|
def build_hash_index(samples: list[dict], bucket: str, prefix: str,
|
|
322
322
|
source_path: str = "images") -> pa.Table:
|
|
323
323
|
now = time.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
324
|
+
if not samples:
|
|
325
|
+
return pa.table({
|
|
326
|
+
"sample_id": pa.array([], type=pa.string()),
|
|
327
|
+
"uri": pa.array([], type=pa.string()),
|
|
328
|
+
"content_hash": pa.array([], type=pa.string()),
|
|
329
|
+
"size": pa.array([], type=pa.int64()),
|
|
330
|
+
"last_modified": pa.array([], type=pa.string()),
|
|
331
|
+
"version_id": pa.array([], type=pa.string()),
|
|
332
|
+
"etag": pa.array([], type=pa.string()),
|
|
333
|
+
"source_kind": pa.array([], type=pa.string()),
|
|
334
|
+
})
|
|
324
335
|
return pa.table({
|
|
325
336
|
"sample_id": [s["sample_id"] for s in samples],
|
|
326
337
|
"uri": [
|
|
@@ -343,6 +354,15 @@ def build_mask_hash_index(masks: list[dict], bucket: str, prefix: str) -> pa.Tab
|
|
|
343
354
|
|
|
344
355
|
|
|
345
356
|
def build_sample_manifests(samples: list[dict]) -> pa.Table:
|
|
357
|
+
if not samples:
|
|
358
|
+
return pa.table({
|
|
359
|
+
"sample_id": pa.array([], type=pa.string()),
|
|
360
|
+
"source_kind": pa.array([], type=pa.string()),
|
|
361
|
+
"primary_uri": pa.array([], type=pa.string()),
|
|
362
|
+
"files_json": pa.array([], type=pa.string()),
|
|
363
|
+
"content_hash": pa.array([], type=pa.string()),
|
|
364
|
+
"n_files": pa.array([], type=pa.int32()),
|
|
365
|
+
})
|
|
346
366
|
return pa.table({
|
|
347
367
|
"sample_id": [s["sample_id"] for s in samples],
|
|
348
368
|
"source_kind": [s["source_kind"] for s in samples],
|
|
@@ -364,6 +384,15 @@ def build_samples_metadata(samples: list[dict],
|
|
|
364
384
|
image samples become nulls. This keeps the manifest image-led while allowing
|
|
365
385
|
clinical/outcome variables to travel with the dataset.
|
|
366
386
|
"""
|
|
387
|
+
if not samples:
|
|
388
|
+
base = pa.table({
|
|
389
|
+
"sample_id": pa.array([], type=pa.string()),
|
|
390
|
+
"source_kind": pa.array([], type=pa.string()),
|
|
391
|
+
"n_files": pa.array([], type=pa.int32()),
|
|
392
|
+
})
|
|
393
|
+
if extra_metadata is None:
|
|
394
|
+
return base
|
|
395
|
+
return _left_join_metadata(base, extra_metadata)
|
|
367
396
|
base = pa.table({
|
|
368
397
|
"sample_id": [s["sample_id"] for s in samples],
|
|
369
398
|
"source_kind": [s["source_kind"] for s in samples],
|
|
@@ -294,7 +294,9 @@ else
|
|
|
294
294
|
fi
|
|
295
295
|
|
|
296
296
|
mc version enable "local/${BUCKET_NAME}"
|
|
297
|
-
|
|
297
|
+
mkdir -p /tmp/dsimaging-init
|
|
298
|
+
: > /tmp/dsimaging-init/.keep
|
|
299
|
+
mc cp /tmp/dsimaging-init/.keep "local/${BUCKET_NAME}/datasets/.keep" 2>/dev/null || true
|
|
298
300
|
mc event add "local/${BUCKET_NAME}" arn:minio:sqs::DSIMAGING:webhook \
|
|
299
301
|
--event put,delete \
|
|
300
302
|
--prefix "datasets/" \
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|