dsimaging-admin 0.1.0__py3-none-any.whl

dsimaging_admin/__init__.py

@@ -0,0 +1,3 @@
+"""dsimaging-admin: Admin CLI for managing medical imaging datasets in S3/MinIO."""
+
+__version__ = "0.1.0"

dsimaging_admin/cli.py

@@ -0,0 +1,428 @@
+"""dsimaging-admin CLI."""
+
+import json
+import os
+import sys
+import tempfile
+from urllib.parse import quote
+
+import click
+
+from . import __version__
+from .s3 import create_client, list_datasets, list_objects
+from .manifest import (
+    scan_images, scan_s3_images, validate_dataset_id, generate_manifest,
+    write_manifest_yaml, build_hash_index, build_sample_manifests,
+    build_samples_metadata,
+)
+
+
+CONFIG_PATH = os.path.expanduser("~/.dsimaging.yaml")
+
+
+def _load_config() -> dict:
+    """Load config from ~/.dsimaging.yaml if it exists."""
+    if not os.path.exists(CONFIG_PATH):
+        return {}
+    try:
+        import yaml
+        with open(CONFIG_PATH) as f:
+            data = yaml.safe_load(f) or {}
+        return data.get("default", data)
+    except Exception:
+        return {}
+
+
+def _default(envvar: str, config_key: str, fallback: str) -> str:
+    """Resolve CLI defaults as env var > config file > fallback."""
+    val = os.environ.get(envvar, "")
+    if val:
+        return val
+    cfg = _load_config()
+    val = cfg.get(config_key, "")
+    if val:
+        return str(val)
+    return fallback
+
+
+@click.group()
+@click.version_option(__version__)
+@click.option("--endpoint", default=_default("DSIMAGING_ENDPOINT", "endpoint", "http://127.0.0.1:9000"),
+              help="S3/MinIO endpoint URL")
+@click.option("--access-key", default=_default("DSIMAGING_ACCESS_KEY", "access_key", "minioadmin"),
+              help="S3 access key")
+@click.option("--secret-key", default=_default("DSIMAGING_SECRET_KEY", "secret_key", "minioadmin123"),
+              help="S3 secret key")
+@click.option("--bucket", default=_default("DSIMAGING_BUCKET", "bucket", "imaging-data"),
+              help="S3 bucket name")
+@click.option("--region", default=_default("DSIMAGING_REGION", "region", ""),
+              help="S3 region (empty for MinIO)")
+@click.pass_context
+def main(ctx, endpoint, access_key, secret_key, bucket, region):
+    """Admin CLI for managing medical imaging datasets in S3/MinIO.
+
+    Configuration priority: CLI flags > environment variables > ~/.dsimaging.yaml
+    """
+    ctx.ensure_object(dict)
+    ctx.obj["s3"] = create_client(endpoint, access_key, secret_key, region)
+    ctx.obj["bucket"] = bucket
+    ctx.obj["endpoint"] = endpoint
+    ctx.obj["access_key"] = access_key
+    ctx.obj["secret_key"] = secret_key
+    ctx.obj["region"] = region
+
+
+@main.command("init")
+@click.option("--endpoint", prompt="S3/MinIO endpoint", default="http://127.0.0.1:9000")
+@click.option("--bucket", prompt="Bucket name", default="imaging-data")
+@click.option("--access-key", prompt="Access key", default="minioadmin")
+@click.option("--secret-key", prompt="Secret key", hide_input=True, default="minioadmin123")
+@click.option("--region", prompt="Region (empty for MinIO)", default="")
+def init_config(endpoint, bucket, access_key, secret_key, region):
+    """Create ~/.dsimaging.yaml configuration file."""
+    import yaml
+    config = {
+        "default": {
+            "endpoint": endpoint,
+            "bucket": bucket,
+            "access_key": access_key,
+            "secret_key": secret_key,
+            "region": region,
+        }
+    }
+    with open(CONFIG_PATH, "w") as f:
+        yaml.dump(config, f, default_flow_style=False)
+    os.chmod(CONFIG_PATH, 0o600)
+    click.echo(f"Config saved to {CONFIG_PATH}")
+
+
+@main.command()
+@click.option("--dataset-id", required=True, help="Dataset identifier")
+@click.option("--source", required=True, type=click.Path(exists=True),
+              help="Local directory containing images")
+@click.option("--modality", default="unknown", help="Imaging modality (ct, mri, etc.)")
+@click.option("--opal-url", default=None, help="Opal server URL for resource registration")
+@click.option("--opal-token", default=None, envvar="OPAL_TOKEN", help="Opal auth token")
+@click.option("--opal-user", default=None, envvar="OPAL_USER", help="Opal username")
+@click.option("--opal-password", default=None, envvar="OPAL_PASSWORD", help="Opal password")
+@click.option("--opal-project", default="IMAGING", help="Opal project name")
+@click.option("--opal-resource", default=None, help="Opal resource name (defaults to dataset_id)")
+@click.option("--opal-replace", is_flag=True, help="Replace an existing Opal resource")
+@click.option("--opal-insecure", is_flag=True, help="Disable TLS certificate verification for Opal")
+@click.pass_context
+def publish(ctx, dataset_id, source, modality, opal_url, opal_token, opal_user,
+            opal_password, opal_project, opal_resource, opal_replace,
+            opal_insecure):
+    """Publish a local dataset to S3/MinIO.
+
+    Scans images, computes hashes, generates manifests and indexes,
+    uploads everything to S3, and optionally registers as a DataSHIELD resource.
+    """
+    try:
+        validate_dataset_id(dataset_id)
+    except ValueError as e:
+        raise click.ClickException(str(e))
+
+    s3 = ctx.obj["s3"]
+    bucket = ctx.obj["bucket"]
+    prefix = f"datasets/{dataset_id}"
+
+    click.echo(f"Publishing dataset: {dataset_id}")
+    click.echo(f"  Source: {os.path.abspath(source)}")
+
+    # 1. Scan images
+    click.echo("  Scanning images...")
+    samples = scan_images(source)
+    if not samples:
+        click.echo("  ERROR: No image files found.", err=True)
+        sys.exit(1)
+    click.echo(f"  Found {len(samples)} samples")
+
+    # 2. Upload images
+    click.echo("  Uploading images to S3...")
+    for sample in samples:
+        if sample["source_kind"] == "single_file":
+            key = f"{prefix}/source/images/{sample['primary_filename']}"
+            s3.upload_file(sample["local_path"], bucket, key)
+        elif sample["source_kind"] == "dicom_series":
+            base_dir = os.path.dirname(sample["local_path"])
+            for f_info in sample["files"]:
+                local = os.path.join(base_dir, f_info["path"])
+                key = f"{prefix}/source/images/{f_info['path']}"
+                s3.upload_file(local, bucket, key)
+    click.echo(f"  Uploaded to s3://{bucket}/{prefix}/source/images/")
+
+    # 3. Generate and upload indexes
+    _write_dataset_artifacts(s3, bucket, prefix, dataset_id, modality, samples)
+
+    resource_url = _resource_url(dataset_id, ctx.obj["endpoint"], bucket,
+                                 prefix, ctx.obj.get("region", ""))
+    if opal_url:
+        click.echo("  Registering Opal resource...")
+        _register_opal_resource(
+            opal_url=opal_url,
+            project=opal_project,
+            name=opal_resource or dataset_id,
+            resource_url=resource_url,
+            access_key=ctx.obj["access_key"],
+            secret_key=ctx.obj["secret_key"],
+            token=opal_token,
+            username=opal_user,
+            password=opal_password,
+            replace=opal_replace,
+            verify=not opal_insecure,
+        )
+
+    # 4. Summary
+    endpoint = ctx.obj["endpoint"]
+    click.echo("")
+    click.echo(click.style(f"Dataset '{dataset_id}' published!", fg="green", bold=True))
+    click.echo(f"  Location: s3://{bucket}/{prefix}/")
+    click.echo(f"  Samples:  {len(samples)}")
+    click.echo("")
+    click.echo("  To use in R:")
+    click.echo(f'    ds.radiomics.extract(conns, dataset_id = "{dataset_id}", ...)')
+    click.echo("")
+    click.echo("  DataSHIELD resource config:")
+    click.echo(f"    URL:      {resource_url}")
+    click.echo(f"    Endpoint: {endpoint}")
+    click.echo(f"    Bucket:   {bucket}")
+    click.echo(f"    Prefix:   {prefix}")
+
+
+@main.command("list")
+@click.pass_context
+def list_cmd(ctx):
+    """List published datasets."""
+    datasets = list_datasets(ctx.obj["s3"], ctx.obj["bucket"])
+    if not datasets:
+        click.echo("No datasets found.")
+        return
+    click.echo(f"Datasets in s3://{ctx.obj['bucket']}/datasets/:")
+    for ds in datasets:
+        status_color = "green" if ds["status"] == "published" else "yellow"
+        click.echo(f"  {ds['dataset_id']} [{click.style(ds['status'], fg=status_color)}]")
+
+
+@main.command()
+@click.pass_context
+def doctor(ctx):
+    """Check system health."""
+    s3 = ctx.obj["s3"]
+    bucket = ctx.obj["bucket"]
+
+    click.echo("dsimaging-admin health check")
+    click.echo("=" * 40)
+
+    # 1. Connectivity
+    click.echo("\n1. S3 connectivity:")
+    try:
+        s3.list_buckets()
+        click.echo(click.style("   OK", fg="green") + ": Connected")
+    except Exception as e:
+        click.echo(click.style("   FAIL", fg="red") + f": {e}")
+        return
+
+    # 2. Bucket
+    click.echo(f"\n2. Bucket '{bucket}':")
+    try:
+        s3.head_bucket(Bucket=bucket)
+        click.echo(click.style("   OK", fg="green") + ": Exists")
+    except Exception:
+        click.echo(click.style("   FAIL", fg="red") + ": Not found")
+        click.echo(f"   Create with: aws s3 mb s3://{bucket} --endpoint-url ...")
+        return
+
+    # 3. Versioning
+    click.echo("\n3. Versioning:")
+    try:
+        resp = s3.get_bucket_versioning(Bucket=bucket)
+        status = resp.get("Status", "Disabled")
+        if status == "Enabled":
+            click.echo(click.style("   OK", fg="green") + ": Enabled")
+        else:
+            click.echo(click.style("   WARN", fg="yellow") + f": {status}")
+    except Exception as e:
+        click.echo(click.style("   FAIL", fg="red") + f": {e}")
+
+    # 4. Datasets
+    click.echo("\n4. Datasets:")
+    datasets = list_datasets(s3, bucket)
+    for ds in datasets:
+        color = "green" if ds["status"] == "published" else "yellow"
+        click.echo(f"   {click.style(ds['status'].upper(), fg=color)}: {ds['dataset_id']}")
+    if not datasets:
+        click.echo("   (none)")
+
+    click.echo(f"\n5. Summary: {len(datasets)} dataset(s)")
+
+
+@main.command()
+@click.option("--dataset-id", required=True)
+@click.pass_context
+def rescan(ctx, dataset_id):
+    """Re-scan and update indexes for a dataset."""
+    s3 = ctx.obj["s3"]
+    bucket = ctx.obj["bucket"]
+    prefix = f"datasets/{dataset_id}"
+
+    try:
+        validate_dataset_id(dataset_id)
+    except ValueError as e:
+        raise click.ClickException(str(e))
+
+    click.echo(f"Rescanning: {dataset_id}")
+
+    objects = list_objects(s3, bucket, f"{prefix}/source/images/")
+    click.echo(f"  Found {len(objects)} objects under source/images/")
+
+    click.echo("  Computing hashes and rebuilding dataset artifacts...")
+    samples = scan_s3_images(s3, bucket, prefix, objects)
+    if not samples:
+        raise click.ClickException("No supported image objects found.")
+
+    modality = _existing_modality(s3, bucket, prefix, fallback="unknown")
+    _write_dataset_artifacts(s3, bucket, prefix, dataset_id, modality, samples)
+
+    click.echo(f"  Index updated: {len(samples)} samples")
+    click.echo(click.style("Rescan complete.", fg="green"))
+
+
+def _write_dataset_artifacts(s3, bucket: str, prefix: str, dataset_id: str,
+                             modality: str, samples: list[dict]) -> None:
+    import pyarrow.parquet as pq
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        click.echo("  Building content hash index...")
+        idx = build_hash_index(samples, bucket, prefix)
+        idx_path = os.path.join(tmpdir, "content_hash_index.parquet")
+        pq.write_table(idx, idx_path)
+        s3.upload_file(idx_path, bucket, f"{prefix}/indexes/content_hash_index.parquet")
+
+        click.echo("  Building sample manifests...")
+        sm = build_sample_manifests(samples)
+        sm_path = os.path.join(tmpdir, "sample_manifests.parquet")
+        pq.write_table(sm, sm_path)
+        s3.upload_file(sm_path, bucket, f"{prefix}/metadata/sample_manifests.parquet")
+
+        click.echo("  Building samples metadata...")
+        meta = build_samples_metadata(samples)
+        meta_path = os.path.join(tmpdir, "samples.parquet")
+        pq.write_table(meta, meta_path)
+        s3.upload_file(meta_path, bucket, f"{prefix}/metadata/samples.parquet")
+
+        click.echo("  Building manifest...")
+        manifest = generate_manifest(dataset_id, bucket, prefix, modality)
+        manifest_path = os.path.join(tmpdir, "manifest.yaml")
+        write_manifest_yaml(manifest, manifest_path)
+        s3.upload_file(manifest_path, bucket, f"{prefix}/manifest.yaml")
+
+
+def _existing_modality(s3, bucket: str, prefix: str, fallback: str) -> str:
+    try:
+        import yaml
+        response = s3.get_object(Bucket=bucket, Key=f"{prefix}/manifest.yaml")
+        body = response["Body"]
+        try:
+            manifest = yaml.safe_load(body.read()) or {}
+        finally:
+            body.close()
+        return manifest.get("modality") or fallback
+    except Exception:
+        return fallback
+
+
+def _resource_url(dataset_id: str, endpoint: str, bucket: str, prefix: str,
+                  region: str = "") -> str:
+    parts = [
+        f"endpoint={quote(endpoint, safe='')}",
+        f"bucket={quote(bucket, safe='')}",
+        f"prefix={quote(prefix, safe='')}",
+    ]
+    if region:
+        parts.append(f"region={quote(region, safe='')}")
+    return f"imaging+dataset://{dataset_id}?" + "&".join(parts)
+
+
+def _register_opal_resource(opal_url: str, project: str, name: str,
+                            resource_url: str, access_key: str,
+                            secret_key: str, token: str | None,
+                            username: str | None, password: str | None,
+                            replace: bool, verify: bool) -> None:
+    try:
+        import requests
+    except ImportError as e:
+        raise click.ClickException("requests is required for Opal registration") from e
+
+    if not verify:
+        import urllib3
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    session = requests.Session()
+    session.verify = verify
+    headers = {"Accept": "application/json"}
+    if token:
+        headers["X-Opal-Auth"] = token
+    elif username and password:
+        session.auth = (username, password)
+    else:
+        raise click.ClickException(
+            "Provide --opal-token or --opal-user/--opal-password for Opal registration."
+        )
+
+    base = opal_url.rstrip("/")
+
+    def request(method: str, path: str, **kwargs):
+        response = session.request(method, f"{base}/ws/{path.lstrip('/')}",
+                                   headers=headers, timeout=30, **kwargs)
+        if response.status_code >= 400:
+            raise click.ClickException(
+                f"Opal {method} {path} failed with {response.status_code}: "
+                f"{response.text[:500]}"
+            )
+        return response
+
+    project_resp = session.get(f"{base}/ws/project/{project}", headers=headers,
+                               timeout=30, verify=verify)
+    if project_resp.status_code == 404:
+        request("POST", "projects", json={"name": project, "title": project})
+    elif project_resp.status_code >= 400:
+        raise click.ClickException(
+            f"Opal project check failed with {project_resp.status_code}: "
+            f"{project_resp.text[:500]}"
+        )
+
+    resource_resp = session.get(f"{base}/ws/project/{project}/resource/{name}",
+                                headers=headers, timeout=30, verify=verify)
+    if resource_resp.status_code == 200:
+        if not replace:
+            click.echo(f"  Opal resource {project}.{name} already exists; keeping it")
+            return
+        request("DELETE", f"project/{project}/resource/{name}")
+    elif resource_resp.status_code != 404:
+        raise click.ClickException(
+            f"Opal resource check failed with {resource_resp.status_code}: "
+            f"{resource_resp.text[:500]}"
+        )
+
+    parameters = {"url": resource_url, "format": None, "_package": None}
+    credentials = {
+        "identity": access_key,
+        "identifier": access_key,
+        "secret": secret_key,
+    }
+    payload = {
+        "provider": "resourcer",
+        "factory": "default",
+        "project": project,
+        "name": name,
+        "description": "dsimaging-store dataset",
+        "parameters": json.dumps(parameters),
+        "credentials": json.dumps(credentials),
+    }
+    request("POST", f"project/{project}/resources", json=payload)
+    click.echo(click.style(f"  Registered Opal resource {project}.{name}", fg="green"))
+
+
+if __name__ == "__main__":
+    main()

dsimaging_admin/hashing.py

@@ -0,0 +1,35 @@
+"""Content hashing utilities."""
+
+import hashlib
+import os
+
+HASH_CHUNK = 65536
+IMAGE_EXTENSIONS = frozenset({
+    ".nii.gz", ".nii", ".nrrd", ".mha", ".mhd", ".dcm",
+    ".svs", ".tif", ".tiff", ".png", ".jpg",
+})
+
+
+def sha256_file(path: str) -> str:
+    """Streaming SHA-256 of a file."""
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        while True:
+            chunk = f.read(HASH_CHUNK)
+            if not chunk:
+                break
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def is_image_file(filename: str) -> bool:
+    lower = filename.lower()
+    return any(lower.endswith(ext) for ext in IMAGE_EXTENSIONS)
+
+
+def sample_id_from_filename(filename: str) -> str:
+    """Strip known extensions to get sample_id."""
+    for ext in sorted(IMAGE_EXTENSIONS, key=len, reverse=True):
+        if filename.lower().endswith(ext):
+            return filename[: -len(ext)]
+    return os.path.splitext(filename)[0]

dsimaging_admin/manifest.py

@@ -0,0 +1,256 @@
+"""Manifest and index generation."""
+
+import hashlib
+import json
+import os
+import re
+import time
+
+import pyarrow as pa
+import yaml
+
+from .hashing import sha256_file, is_image_file, sample_id_from_filename
+
+DATASET_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
+
+
+def validate_dataset_id(dataset_id: str) -> str:
+    """Validate and return a dsimaging dataset identifier."""
+    if not DATASET_ID_RE.match(dataset_id or ""):
+        raise ValueError(
+            "dataset_id must match ^[a-z0-9][a-z0-9._-]*$ "
+            "(lowercase letters, digits, dot, underscore and dash)"
+        )
+    return dataset_id
+
+
+def scan_images(source_dir: str) -> list[dict]:
+    """Scan a local directory for image files and compute hashes.
+
+    Returns a list of sample dicts with:
+        sample_id, source_kind, primary_filename, files, content_hash, size, local_path
+    """
+    samples = []
+    images_dir = _find_images_dir(source_dir)
+    if not images_dir:
+        return samples
+
+    for entry in sorted(os.listdir(images_dir)):
+        filepath = os.path.join(images_dir, entry)
+
+        if os.path.isfile(filepath) and is_image_file(entry):
+            samples.append({
+                "sample_id": sample_id_from_filename(entry),
+                "source_kind": "single_file",
+                "primary_filename": entry,
+                "uri_path": entry,
+                "files": [{"path": entry, "role": "primary"}],
+                "content_hash": sha256_file(filepath),
+                "size": os.path.getsize(filepath),
+                "local_path": filepath,
+            })
+        elif os.path.isdir(filepath):
+            dcm_files = sorted(
+                f for f in os.listdir(filepath) if f.lower().endswith(".dcm")
+            )
+            if dcm_files:
+                h = hashlib.sha256()
+                total_size = 0
+                for dcm in dcm_files:
+                    dcm_path = os.path.join(filepath, dcm)
+                    h.update(sha256_file(dcm_path).encode())
+                    total_size += os.path.getsize(dcm_path)
+                samples.append({
+                    "sample_id": entry,
+                    "source_kind": "dicom_series",
+                    "primary_filename": None,
+                    "uri_path": f"{entry}/",
+                    "files": [{"path": f"{entry}/{f}", "role": "slice"} for f in dcm_files],
+                    "content_hash": h.hexdigest(),
+                    "size": total_size,
+                    "local_path": filepath,
+                })
+
+    return samples
+
+
+def scan_s3_images(s3, bucket: str, prefix: str, objects: list[dict]) -> list[dict]:
+    """Build sample records from S3 objects under ``<prefix>/source/images``.
+
+    Hashes are computed by streaming objects through temporary files. Single-file
+    samples and one-directory DICOM series use the same sample model as
+    ``scan_images()`` so publish, rescan and the store controller converge on
+    identical parquet schemas.
+    """
+    root = f"{prefix.rstrip('/')}/source/images/"
+    single_files = []
+    dicom_groups = {}
+
+    for obj in objects:
+        key = obj["key"]
+        if not key.startswith(root):
+            continue
+        rel = key[len(root):]
+        if not rel or rel.endswith("/"):
+            continue
+        filename = rel.rsplit("/", 1)[-1]
+        if not is_image_file(filename):
+            continue
+        if "/" in rel and filename.lower().endswith(".dcm"):
+            sample_id = rel.split("/", 1)[0]
+            dicom_groups.setdefault(sample_id, []).append((rel, obj))
+        else:
+            single_files.append((rel, obj))
+
+    samples = []
+    for rel, obj in sorted(single_files, key=lambda item: item[0]):
+        content_hash = _sha256_s3_object(s3, bucket, obj["key"])
+        filename = rel.rsplit("/", 1)[-1]
+        samples.append({
+            "sample_id": sample_id_from_filename(filename),
+            "source_kind": "single_file",
+            "primary_filename": filename,
+            "uri_path": rel,
+            "files": [{"path": rel, "role": "primary"}],
+            "content_hash": content_hash,
+            "size": int(obj.get("size", 0)),
+            "last_modified": obj.get("last_modified"),
+            "version_id": obj.get("version_id"),
+            "etag": obj.get("etag"),
+        })
+
+    for sample_id in sorted(dicom_groups):
+        h = hashlib.sha256()
+        total_size = 0
+        files = []
+        last_modified = None
+        etags = []
+        for rel, obj in sorted(dicom_groups[sample_id], key=lambda item: item[0]):
+            content_hash = _sha256_s3_object(s3, bucket, obj["key"])
+            h.update(content_hash.encode())
+            total_size += int(obj.get("size", 0))
+            last_modified = obj.get("last_modified") or last_modified
+            if obj.get("etag"):
+                etags.append(obj["etag"])
+            files.append({"path": rel, "role": "slice"})
+        samples.append({
+            "sample_id": sample_id,
+            "source_kind": "dicom_series",
+            "primary_filename": None,
+            "uri_path": f"{sample_id}/",
+            "files": files,
+            "content_hash": h.hexdigest(),
+            "size": total_size,
+            "last_modified": last_modified,
+            "version_id": None,
+            "etag": ",".join(etags) if etags else None,
+        })
+
+    return sorted(samples, key=lambda sample: sample["sample_id"])
+
+
+def generate_manifest(dataset_id: str, bucket: str, prefix: str,
+                      modality: str = "unknown") -> dict:
+    """Generate a manifest dict for a dataset."""
+    validate_dataset_id(dataset_id)
+    return {
+        "schema_version": 1,
+        "dataset_id": dataset_id,
+        "modality": modality,
+        "assets": {
+            "images": {
+                "uri": f"s3://{bucket}/{prefix}/source/images/",
+                "kind": "image_root",
+            },
+        },
+        "metadata": {
+            "uri": f"s3://{bucket}/{prefix}/metadata/samples.parquet",
+            "format": "parquet",
+        },
+        "content_hash_index": {
+            "uri": f"s3://{bucket}/{prefix}/indexes/content_hash_index.parquet",
+            "format": "parquet",
+        },
+        "sample_manifests": {
+            "uri": f"s3://{bucket}/{prefix}/metadata/sample_manifests.parquet",
+            "format": "parquet",
+        },
+    }
+
+
+def write_manifest_yaml(manifest: dict, path: str):
+    with open(path, "w") as f:
+        yaml.dump(manifest, f, default_flow_style=False, sort_keys=False)
+
+
+def build_hash_index(samples: list[dict], bucket: str, prefix: str) -> pa.Table:
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ")
+    return pa.table({
+        "sample_id": [s["sample_id"] for s in samples],
+        "uri": [
+            f"s3://{bucket}/{prefix}/source/images/{s.get('uri_path') or s['primary_filename']}"
+            if s.get("uri_path") or s["primary_filename"]
+            else f"s3://{bucket}/{prefix}/source/images/{s['sample_id']}/"
+            for s in samples
+        ],
+        "content_hash": [s["content_hash"] for s in samples],
+        "size": pa.array([s["size"] for s in samples], type=pa.int64()),
+        "last_modified": [s.get("last_modified") or now for s in samples],
+        "version_id": pa.array([s.get("version_id") for s in samples], type=pa.string()),
+        "etag": pa.array([s.get("etag") for s in samples], type=pa.string()),
+        "source_kind": [s["source_kind"] for s in samples],
+    })
+
+
+def build_sample_manifests(samples: list[dict]) -> pa.Table:
+    return pa.table({
+        "sample_id": [s["sample_id"] for s in samples],
+        "source_kind": [s["source_kind"] for s in samples],
+        "primary_uri": pa.array(
+            [s["primary_filename"] for s in samples], type=pa.string()
+        ),
+        "files_json": [json.dumps(s["files"]) for s in samples],
+        "content_hash": [s["content_hash"] for s in samples],
+        "n_files": pa.array([len(s["files"]) for s in samples], type=pa.int32()),
+    })
+
+
+def build_samples_metadata(samples: list[dict]) -> pa.Table:
+    return pa.table({
+        "sample_id": [s["sample_id"] for s in samples],
+        "source_kind": [s["source_kind"] for s in samples],
+        "n_files": pa.array([len(s["files"]) for s in samples], type=pa.int32()),
+    })
+
+
+def _find_images_dir(source_dir: str) -> str | None:
+    """Find the directory containing image files."""
+    for candidate in ["images", "source/images", "."]:
+        d = os.path.join(source_dir, candidate)
+        if os.path.isdir(d) and _contains_supported_images(d):
+            return d
+    return None
+
+
+def _contains_supported_images(directory: str) -> bool:
+    for entry in os.listdir(directory):
+        path = os.path.join(directory, entry)
+        if os.path.isfile(path) and is_image_file(entry):
+            return True
+        if os.path.isdir(path):
+            if any(f.lower().endswith(".dcm") for f in os.listdir(path)):
+                return True
+    return False
+
+
+def _sha256_s3_object(s3, bucket: str, key: str) -> str:
+    h = hashlib.sha256()
+    response = s3.get_object(Bucket=bucket, Key=key)
+    body = response["Body"]
+    try:
+        for chunk in iter(lambda: body.read(65536), b""):
+            if chunk:
+                h.update(chunk)
+    finally:
+        body.close()
+    return h.hexdigest()

dsimaging_admin/s3.py

@@ -0,0 +1,64 @@
+"""S3/MinIO client wrapper."""
+
+import boto3
+from botocore.config import Config
+
+
+def create_client(endpoint: str, access_key: str, secret_key: str,
+                  region: str = "") -> boto3.client:
+    """Create a boto3 S3 client.
+
+    For MinIO (IP/localhost endpoints), region defaults to "us-east-1"
+    as a dummy value that boto3 accepts.
+    """
+    effective_region = region if region else "us-east-1"
+    kwargs = {"region_name": effective_region}
+    if endpoint:
+        kwargs["endpoint_url"] = endpoint
+
+    return boto3.client(
+        "s3",
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        config=Config(signature_version="s3v4"),
+        **kwargs,
+    )
+
+
+def list_datasets(s3, bucket: str) -> list[dict]:
+    """List published datasets in the bucket."""
+    datasets = []
+    paginator = s3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=bucket, Prefix="datasets/", Delimiter="/"):
+        for cp in page.get("CommonPrefixes", []):
+            ds_id = cp["Prefix"].strip("/").split("/")[-1]
+            has_manifest = _object_exists(s3, bucket, f"datasets/{ds_id}/manifest.yaml")
+            datasets.append({
+                "dataset_id": ds_id,
+                "status": "published" if has_manifest else "incomplete",
+            })
+    return datasets
+
+
+def list_objects(s3, bucket: str, prefix: str) -> list[dict]:
+    """List all objects under a prefix (with pagination)."""
+    objects = []
+    paginator = s3.get_paginator("list_objects_v2")
+    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+        for obj in page.get("Contents", []):
+            objects.append({
+                "key": obj["Key"],
+                "size": obj["Size"],
+                "last_modified": obj["LastModified"].isoformat(),
+                "etag": obj.get("ETag", "").strip('"') or None,
+                "version_id": None,
+            })
+    return objects
+
+
+def _object_exists(s3, bucket: str, key: str) -> bool:
+    try:
+        s3.head_object(Bucket=bucket, Key=key)
+        return True
+    except Exception:
+        return False

dsimaging_admin-0.1.0.dist-info/METADATA

@@ -0,0 +1,130 @@
+Metadata-Version: 2.4
+Name: dsimaging-admin
+Version: 0.1.0
+Summary: Admin CLI for managing medical imaging datasets in S3/MinIO for DataSHIELD
+Project-URL: Homepage, https://davidsarrat.com
+Project-URL: Repository, https://github.com/isglobal-brge/dsimaging-admin
+Project-URL: Issues, https://github.com/isglobal-brge/dsimaging-admin/issues
+Project-URL: Documentation, https://github.com/isglobal-brge/dsimaging-admin#readme
+Author-email: David Sarrat Gonzalez <david.sarrat@isglobal.org>, Juan R Gonzalez <juanr.gonzalez@isglobal.org>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: datashield,dicom,medical-imaging,minio,radiomics,s3
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Healthcare Industry
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
+Classifier: Topic :: System :: Archiving
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.9
+Requires-Dist: boto3>=1.28.0
+Requires-Dist: click>=8.0.0
+Requires-Dist: pyarrow>=14.0.0
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.31.0
+Description-Content-Type: text/markdown
+
+# dsimaging-admin
+
+Admin CLI for managing medical imaging datasets in S3/MinIO for DataSHIELD.
+
+## Install
+
+```bash
+pip install dsimaging-admin
+```
+
+## Quick start
+
+```bash
+# 1. Publish a local dataset to MinIO
+dsimaging-admin --endpoint http://minio:9000 publish \
+  --dataset-id lung_ct_v1 \
+  --source /data/lung_ct \
+  --modality ct
+
+# 2. List published datasets
+dsimaging-admin --endpoint http://minio:9000 list
+
+# 3. Check health
+dsimaging-admin --endpoint http://minio:9000 doctor
+
+# 4. Re-scan after adding images
+dsimaging-admin --endpoint http://minio:9000 rescan --dataset-id lung_ct_v1
+```
+
+For repeated use, create `~/.dsimaging.yaml` once:
+
+```bash
+dsimaging-admin init
+```
+
+## What `publish` does
+
+1. Scans your local image directory (NIfTI, DICOM, NRRD, etc.)
+2. Computes SHA-256 content hash for every file
+3. Uploads images to `s3://<bucket>/datasets/<dataset_id>/source/images/`
+4. Generates and uploads:
+   - `manifest.yaml` (dataset descriptor)
+   - `content_hash_index.parquet` (dedup index)
+   - `sample_manifests.parquet` (multi-file sample support)
+   - `samples.parquet` (basic metadata)
+5. Optionally registers the dataset as an Opal resource
+6. Prints the DataSHIELD resource configuration
+
+`publish` can register the resource directly in Opal:
+
+```bash
+dsimaging-admin --endpoint http://localhost:9000 publish \
+  --dataset-id lung_ct_v1 \
+  --source /data/lung_ct \
+  --modality ct \
+  --opal-url https://opal.example.org \
+  --opal-user administrator \
+  --opal-password "$OPAL_PASSWORD" \
+  --opal-project IMAGING
+```
+
+## Environment variables
+
+| Variable | Default | Description |
+|---|---|---|
+| `DSIMAGING_ENDPOINT` | `http://127.0.0.1:9000` | S3/MinIO endpoint |
+| `DSIMAGING_ACCESS_KEY` | `minioadmin` | S3 access key |
+| `DSIMAGING_SECRET_KEY` | `minioadmin123` | S3 secret key |
+| `DSIMAGING_BUCKET` | `imaging-data` | Bucket name |
+| `DSIMAGING_REGION` | (empty) | S3 region |
+| `OPAL_TOKEN` | (empty) | Optional Opal token for `publish --opal-url` |
+| `OPAL_USER` | (empty) | Optional Opal username for `publish --opal-url` |
+| `OPAL_PASSWORD` | (empty) | Optional Opal password for `publish --opal-url` |
+
+## Rescan
+
+`rescan --dataset-id <id>` rebuilds `content_hash_index.parquet`,
+`sample_manifests.parquet`, `samples.parquet` and `manifest.yaml` from the
+current contents of `source/images/`. This is the same contract maintained
+automatically by the dsimaging-store controller when MinIO webhooks are enabled.
+
+## Dataset layout in S3
+
+```
+s3://<bucket>/datasets/<dataset_id>/
+  manifest.yaml
+  metadata/
+    samples.parquet
+    sample_manifests.parquet
+  indexes/
+    content_hash_index.parquet
+  source/
+    images/
+  derived/
+  qc/
+```

dsimaging_admin-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+dsimaging_admin/__init__.py,sha256=8jQ0qfys9B9xcCqYliau-5rlQko-zKVqSXWrRtH7iwo,107
+dsimaging_admin/cli.py,sha256=GcQZucbX6BECasEt0bWPqNqRc5kbc_o5HRODNNmaIB8,16143
+dsimaging_admin/hashing.py,sha256=Wso0w-RSRmmjllUURL9DX89lE5BgbRNHOVbUVgCrVuc,937
+dsimaging_admin/manifest.py,sha256=WeG5wLfUAN5p5_81fwQohvHSJx7JPA4wjfj0TjMYI0M,9289
+dsimaging_admin/s3.py,sha256=qsu_Jl6G07BX8pe4RhgKPfLt437TM6_EFNQZp4u87LA,2124
+dsimaging_admin-0.1.0.dist-info/METADATA,sha256=jYixpn0EGzYkIR-WGCmQqjv8UJ-ULLtoduefN3SHi0g,4250
+dsimaging_admin-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+dsimaging_admin-0.1.0.dist-info/entry_points.txt,sha256=3yKyhItp2qS3UPIDZs6rNK3KWXLOvgBqQYjE9blPS2c,61
+dsimaging_admin-0.1.0.dist-info/licenses/LICENSE,sha256=MbaF7Y7CDdVsRbfLvBjhoAcHsPtGIvuv3CA5tLxrisg,1107
+dsimaging_admin-0.1.0.dist-info/RECORD,,

dsimaging_admin-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dsimaging_admin-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dsimaging-admin = dsimaging_admin.cli:main

dsimaging_admin-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 David Sarrat González, Juan R González, ISGlobal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.