ds_crawler 2.7.2__tar.gz → 2.8.0__tar.gz

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ds_crawler
-Version: 2.7.2
+Version: 2.8.0
 Summary: Regex-based dataset metadata crawler and indexer
 Requires-Python: >=3.9
 Requires-Dist: euler-dataset-contract>=0.1.0

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/README.md

@@ -43,6 +43,31 @@ The files have distinct roles:
 | `.ds_crawler/index.json` | Materialized full dataset index. No duplicated head/config metadata. |
 | `.ds_crawler/split_<name>.json` | Named split artifact with its own contract, provenance, and filtered `index` node. |
 
+When several logical modalities share one physical root or zip, store each
+metadata set under a scope:
+
+```text
+dataset_root/
+├── .ds_crawler/
+│   ├── scopes.json
+│   ├── rgb/
+│   │   ├── dataset-head.json
+│   │   ├── ds-crawler.json
+│   │   └── index.json
+│   └── camera_extrinsics/
+│       ├── dataset-head.json
+│       ├── ds-crawler.json
+│       └── index.json
+├── calib.json
+└── ...
+```
+
+Pass `metadata_scope="rgb"` or `metadata_scope="camera_extrinsics"` to the
+path-based APIs to read/write that scoped artifact set. Scoped writes update
+`.ds_crawler/scopes.json`, a discoverability manifest listing known scopes and
+their artifact filenames. Omitting `metadata_scope` preserves the legacy
+unscoped layout exactly.
+
 The in-memory objects returned by `index_dataset_from_path(...)` and
 `load_dataset_split(...)` are hydrated outputs. They include `head` and
 `indexing` for convenience, even though the on-disk `index.json` only stores
@@ -146,6 +171,21 @@ print(output["index"].keys())
 - `.ds_crawler/ds-crawler.json`
 - `.ds_crawler/index.json`
 
+For a shared physical root, pass a metadata scope:
+
+```python
+rgb = index_dataset_from_path(
+    "/data/muses",
+    save_index=True,
+    metadata_scope="rgb",
+)
+extrinsics = index_dataset_from_path(
+    "/data/muses",
+    save_index=True,
+    metadata_scope="camera_extrinsics",
+)
+```
+
 ### 3. Align multiple modalities
 
 ```python
@@ -237,6 +277,19 @@ print(train_output["split"]["name"])
 print(train_output["execution"]["split"])
 ```
 
+Scoped split artifacts use the same namespace as the index:
+
+```python
+create_dataset_splits(
+    "/data/muses",
+    split_names=["train", "val"],
+    ratios=[80, 20],
+    metadata_scope="rgb",
+)
+
+train_output = load_dataset_split("/data/muses", "train", metadata_scope="rgb")
+```
+
 To reuse an existing partition on a sibling dataset (same qualified file
 IDs), copy the splits over instead of recomputing them:
 
@@ -497,6 +550,9 @@ The main public entry points are re-exported from `ds_crawler`.
 
 Indexing and config:
 
+- `build_crawler_config(...)`
+- `build_dataset_artifacts_from_files(...)`
+- `build_dataset_head(...)`
 - `index_dataset(...)`
 - `index_dataset_from_files(...)`
 - `index_dataset_from_path(...)`
@@ -505,14 +561,22 @@ Indexing and config:
 - `validate_dataset(...)`
 - `validate_output(...)`
 
+`index_dataset(...)`, `index_dataset_from_path(...)`, `load_dataset_config(...)`,
+and `validate_dataset(...)` accept `metadata_scope=...` for scoped metadata.
+`build_dataset_artifacts_from_files(...)` also accepts `metadata_scope=...` and
+returns scoped artifact names plus `scopes.json`.
+
 Dataset metadata:
 
 - `get_dataset_contract(source)`
 - `get_dataset_properties(source)`
 - `extract_dataset_properties(mapping)`
+- `list_metadata_scopes(path)`
 
 `get_dataset_contract(...)` returns a `DatasetHeadContract`. Use
 `contract.get_namespace("euler_train")` to access addon payloads.
+`get_dataset_contract(path, metadata_scope=...)` and
+`get_dataset_properties(path, metadata_scope=...)` read scoped heads.
 
 Traversal and filtering:
 
@@ -535,16 +599,29 @@ Operations:
 - `list_dataset_splits(...)`
 - `load_dataset_split(...)`
 
+Split, copy, align, and extraction operations accept `metadata_scope=...`.
+`copy_dataset(...)`, `copy_dataset_splits(...)`, and `split_dataset(...)` also
+accept source/target scope overrides when the two sides use different scope
+names. `split_datasets(...)` and `extract_datasets(...)` accept per-source or
+per-config scope lists for multi-dataset workflows.
+
 Writers:
 
 - `DatasetWriter(...)`
 - `ZipDatasetWriter(...)`
 
+Both writers accept `metadata_scope=...` and write scoped artifacts plus the
+`scopes.json` manifest.
+
 Migration helpers:
 
 - `migrate_dataset_metadata(...)`
 - `migrate_dataset_zip(...)`
 - `migrate_dataset_zips_in_folder(...)`
+- `migrate_inline_splits(...)`
+
+Migration helpers and `ds-crawler migrate` also accept `metadata_scope=...`
+or `--metadata-scope` for scoped legacy metadata conversion.
 
 ## ZIP support
 

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/ds_crawler/__init__.py

@@ -46,6 +46,7 @@ from .migration import (
     migrate_dataset_zips_in_folder,
 )
 from .writer import DatasetWriter, ZipDatasetWriter
+from .zip_utils import list_metadata_scopes, validate_metadata_scope
 from .layout import (
     EULER_LAYOUT_ADDON,
     EULER_LAYOUT_VERSION,
@@ -86,6 +87,7 @@ __all__ = [
     "index_dataset_from_files",
     "index_dataset_from_path",
     "list_dataset_splits",
+    "list_metadata_scopes",
     "load_dataset_config",
     "load_dataset_split",
     "migrate_dataset_metadata",
@@ -96,6 +98,7 @@ __all__ = [
     "split_qualified_ids",
     "validate_crawler_config",
     "validate_dataset",
+    "validate_metadata_scope",
     "validate_layout_addon",
     "validate_output",
     "extract_dataset_properties",

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/ds_crawler/artifact_builder.py

@@ -15,7 +15,14 @@ from .artifacts import build_index_artifact
 from .config import CONFIG_FILENAME, CRAWLER_CONFIG_KIND, CRAWLER_CONFIG_VERSION, DatasetConfig
 from .parser import index_dataset_from_files
 from .traversal import get_files
-from .zip_utils import DATASET_HEAD_FILENAME, OUTPUT_FILENAME
+from .zip_utils import (
+    DATASET_HEAD_FILENAME,
+    OUTPUT_FILENAME,
+    SCOPES_FILENAME,
+    _scopes_manifest_with,
+    get_metadata_filename,
+    validate_metadata_scope,
+)
 
 
 def _require_mapping(value: Any, context: str) -> dict[str, Any]:
@@ -125,6 +132,7 @@ def build_crawler_config(
     source_path: str = ".",
     head_file: str = DATASET_HEAD_FILENAME,
     prebuilt_index_file: str | None = None,
+    metadata_scope: str | None = None,
 ) -> dict[str, Any]:
     """Build and validate a canonical ``ds-crawler.json`` mapping."""
     indexing = _require_mapping(indexing, "indexing")
@@ -150,6 +158,8 @@ def build_crawler_config(
     }
     if prebuilt_index_file is not None:
         config["source"]["prebuilt_index_file"] = prebuilt_index_file
+    if metadata_scope is not None:
+        config["metadata_scope"] = validate_metadata_scope(metadata_scope)
 
     runtime_config = dict(config)
     runtime_config["head"] = deepcopy(head)
@@ -172,10 +182,20 @@ def build_dataset_artifacts_from_files(
     strict: bool = False,
     sample: int | None = None,
     match_index: dict[str, Any] | None = None,
+    metadata_scope: str | None = None,
 ) -> dict[str, Any]:
     """Build canonical dataset artifacts from a parameter set and file list."""
+    normalized_scope = (
+        validate_metadata_scope(metadata_scope)
+        if metadata_scope is not None
+        else None
+    )
     head = build_dataset_head(dataset=dataset, modality=modality, addons=addons)
-    config = build_crawler_config(head=head, indexing=indexing)
+    config = build_crawler_config(
+        head=head,
+        indexing=indexing,
+        metadata_scope=normalized_scope,
+    )
 
     runtime_config = dict(config)
     runtime_config["head"] = deepcopy(head)
@@ -193,14 +213,33 @@ def build_dataset_artifacts_from_files(
         CONFIG_FILENAME: config,
         OUTPUT_FILENAME: build_index_artifact(output),
     }
+    if normalized_scope is not None:
+        unscoped_artifact_names = list(artifacts)
+        artifacts = {
+            get_metadata_filename(
+                filename,
+                metadata_scope=normalized_scope,
+            ): payload
+            for filename, payload in artifacts.items()
+        }
+        artifacts[SCOPES_FILENAME] = _scopes_manifest_with(
+            None,
+            metadata_scope=normalized_scope,
+            filenames=unscoped_artifact_names,
+        )
+
+    summary = {
+        "dataset_id": output["head"]["dataset"]["id"],
+        "dataset_name": output["head"]["dataset"]["name"],
+        "modality_key": output["head"]["modality"]["key"],
+        "file_count": len(get_files(output)),
+    }
+    if normalized_scope is not None:
+        summary["metadata_scope"] = normalized_scope
+
     return {
         "artifacts": artifacts,
-        "summary": {
-            "dataset_id": output["head"]["dataset"]["id"],
-            "dataset_name": output["head"]["dataset"]["name"],
-            "modality_key": output["head"]["modality"]["key"],
-            "file_count": len(get_files(output)),
-        },
+        "summary": summary,
     }
 
 

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/ds_crawler/artifacts.py

@@ -173,13 +173,21 @@ def load_saved_output(
     dataset_path: str | Path,
     *,
     filename: str = OUTPUT_FILENAME,
+    metadata_scope: str | None = None,
 ) -> dict[str, Any] | None:
     """Load and hydrate a persisted dataset index artifact from disk."""
     dataset_root = Path(dataset_path)
-    index_artifact = read_metadata_json(dataset_root, filename)
+    index_artifact = read_metadata_json(
+        dataset_root,
+        filename,
+        metadata_scope=metadata_scope,
+    )
     if index_artifact is None:
         return None
-    ds_config = load_dataset_config({"path": str(dataset_root)})
+    ds_config = load_dataset_config(
+        {"path": str(dataset_root)},
+        metadata_scope=metadata_scope,
+    )
     return hydrate_index_artifact(index_artifact, ds_config)
 
 
@@ -196,6 +204,7 @@ def load_prebuilt_output(ds_config: DatasetConfig) -> dict[str, Any]:
         index_artifact = read_metadata_json(
             Path(ds_config.dataset_root),
             prebuilt_path.name,
+            metadata_scope=ds_config.metadata_scope,
         )
         if index_artifact is None:
             raise FileNotFoundError(
@@ -210,6 +219,7 @@ def save_output_artifacts(
     output: dict[str, Any],
     *,
     filename: str = OUTPUT_FILENAME,
+    metadata_scope: str | None = None,
 ) -> Path:
     """Persist head/config/index artifacts for a dataset output."""
     dataset_root = Path(dataset_path)
@@ -227,8 +237,11 @@ def save_output_artifacts(
             CONFIG_FILENAME: build_crawler_config_for_output(output),
             filename: build_index_artifact(output),
         },
+        metadata_scope=metadata_scope,
     )
 
     if is_zip_path(dataset_root):
         return dataset_root
+    if metadata_scope is not None:
+        return dataset_root / METADATA_DIR / metadata_scope / filename
     return dataset_root / METADATA_DIR / filename

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/ds_crawler/cli.py

@@ -85,13 +85,27 @@ def _run_index(argv: list[str]) -> int:
         metavar="PATH",
         help="Path to an index.json whose file IDs are used as a filter.",
     )
+    parser.add_argument(
+        "--metadata-scope",
+        type=str,
+        default=None,
+        metavar="SCOPE",
+        help=(
+            "Read/write ds-crawler artifacts below .ds_crawler/SCOPE/ "
+            "instead of the root .ds_crawler directory."
+        ),
+    )
 
     args = parser.parse_args(argv)
 
     setup_logging(args.verbose)
 
     try:
-        config = Config.from_file(args.config, workdir=args.workdir)
+        config = Config.from_file(
+            args.config,
+            workdir=args.workdir,
+            metadata_scope=args.metadata_scope,
+        )
     except FileNotFoundError as e:
         print(f"Error: {e}", file=sys.stderr)
         return 1
@@ -122,7 +136,9 @@ def _run_index(argv: list[str]) -> int:
             parser_instance.write_output(args.output)
             logger.info(f"Output written to: {args.output}")
         else:
-            output_paths = parser_instance.write_outputs_per_dataset()
+            output_paths = parser_instance.write_outputs_per_dataset(
+                metadata_scope=args.metadata_scope,
+            )
             for path in output_paths:
                 logger.info(f"Output written to: {path}")
     except Exception as e:
@@ -165,6 +181,13 @@ def _run_migrate(argv: list[str]) -> int:
         action="store_true",
         help="When used with --scan-zips, only scan the provided directory itself.",
     )
+    parser.add_argument(
+        "--metadata-scope",
+        help=(
+            "Read/write legacy migration artifacts below "
+            ".ds_crawler/SCOPE/."
+        ),
+    )
     parser.add_argument(
         "-v",
         "--verbose",
@@ -188,6 +211,7 @@ def _run_migrate(argv: list[str]) -> int:
                 result = migrate_inline_splits(
                     path,
                     logger=logger,
+                    metadata_scope=args.metadata_scope,
                 )
                 logger.info(
                     "Migrated inline splits %s (splits=%d)",
@@ -202,6 +226,7 @@ def _run_migrate(argv: list[str]) -> int:
                     recursive=not args.top_level_only,
                     write_output=not args.no_index,
                     logger=logger,
+                    metadata_scope=args.metadata_scope,
                 )
                 if result["failed"]:
                     failed = True
@@ -222,6 +247,7 @@ def _run_migrate(argv: list[str]) -> int:
                 path,
                 write_output=not args.no_index,
                 logger=logger,
+                metadata_scope=args.metadata_scope,
             )
         except Exception as exc:
             failed = True
@@ -274,6 +300,27 @@ def _run_copy_splits(argv: list[str]) -> int:
         action="store_true",
         help="Replace existing splits on the target if they share a name.",
     )
+    parser.add_argument(
+        "--metadata-scope",
+        type=str,
+        default=None,
+        metavar="SCOPE",
+        help="Use the same scoped .ds_crawler/SCOPE metadata for source and target.",
+    )
+    parser.add_argument(
+        "--source-metadata-scope",
+        type=str,
+        default=None,
+        metavar="SCOPE",
+        help="Read source splits from .ds_crawler/SCOPE/.",
+    )
+    parser.add_argument(
+        "--target-metadata-scope",
+        type=str,
+        default=None,
+        metavar="SCOPE",
+        help="Write target splits to .ds_crawler/SCOPE/.",
+    )
     parser.add_argument(
         "-v",
         "--verbose",
@@ -291,6 +338,9 @@ def _run_copy_splits(argv: list[str]) -> int:
             args.target,
             split_names=args.splits,
             override=args.override,
+            metadata_scope=args.metadata_scope,
+            source_metadata_scope=args.source_metadata_scope,
+            target_metadata_scope=args.target_metadata_scope,
         )
     except (FileNotFoundError, ValueError) as exc:
         print(f"Error: {exc}", file=sys.stderr)

{ds_crawler-2.7.2 → ds_crawler-2.8.0}/ds_crawler/config.py

@@ -16,7 +16,11 @@ from ._dataset_contract import (
     validate_contract_version,
 )
 from .path_filters import PathFilters
-from .zip_utils import DATASET_HEAD_FILENAME, read_metadata_json
+from .zip_utils import (
+    DATASET_HEAD_FILENAME,
+    read_metadata_json,
+    validate_metadata_scope,
+)
 
 
 CONFIG_FILENAME = "ds-crawler.json"
@@ -81,6 +85,7 @@ class DatasetConfig:
     path_filters: dict[str, Any] | None = None
     prebuilt_index_file: str | None = None
     file_extensions: list[str] | None = None
+    metadata_scope: str | None = None
     compiled_path_filters: PathFilters = field(init=False, repr=False)
 
     def __post_init__(self) -> None:
@@ -167,6 +172,8 @@ class DatasetConfig:
         }
         if self.prebuilt_index_file is not None:
             result["source"]["prebuilt_index_file"] = self.prebuilt_index_file
+        if self.metadata_scope is not None:
+            result["metadata_scope"] = self.metadata_scope
         return result
 
     def _normalize_file_extensions(self) -> None:
@@ -242,11 +249,22 @@ class DatasetConfig:
         workdir: str | Path | None = None,
         dataset_root: str | Path | None = None,
         dataset_head: DatasetHeadContract | dict[str, Any] | None = None,
+        metadata_scope: str | None = None,
     ) -> "DatasetConfig":
         if not isinstance(data, dict):
             raise ValueError("Crawler config must be a JSON object")
 
         _normalize_config_contract(data, "config")
+        raw_metadata_scope = metadata_scope
+        if raw_metadata_scope is None:
+            config_scope = data.get("metadata_scope")
+            if config_scope is not None:
+                raw_metadata_scope = config_scope
+        normalized_metadata_scope = (
+            validate_metadata_scope(raw_metadata_scope)
+            if raw_metadata_scope is not None
+            else None
+        )
 
         source = _require_mapping(data.get("source", {}), "config.source")
         raw_source_path = source.get("path", ".")
@@ -271,10 +289,19 @@ class DatasetConfig:
             if embedded_head is not None:
                 dataset_head = embedded_head
             else:
-                raw_head = read_metadata_json(dataset_root_path, head_file)
+                raw_head = read_metadata_json(
+                    dataset_root_path,
+                    head_file,
+                    metadata_scope=normalized_metadata_scope,
+                )
                 if raw_head is None:
                     raise FileNotFoundError(
                         f"No {head_file} found at: {dataset_root_path}"
+                        + (
+                            f" (metadata_scope={normalized_metadata_scope!r})"
+                            if normalized_metadata_scope is not None
+                            else ""
+                        )
                     )
                 dataset_head = raw_head
 
@@ -375,30 +402,61 @@ class DatasetConfig:
             path_filters=files_cfg.get("path_filters"),
             prebuilt_index_file=prebuilt_index_file,
             file_extensions=file_extensions,
+            metadata_scope=normalized_metadata_scope,
         )
 
 
 def load_dataset_config(
     data: dict[str, Any],
     workdir: str | Path | None = None,
+    *,
+    metadata_scope: str | None = None,
 ) -> DatasetConfig:
     """Load a DatasetConfig, resolving from ``ds-crawler.json`` when needed."""
     if not isinstance(data, dict):
         raise ValueError("Crawler config must be a JSON object")
 
+    raw_metadata_scope = metadata_scope
+    if raw_metadata_scope is None:
+        config_scope = data.get("metadata_scope")
+        if config_scope is not None:
+            raw_metadata_scope = config_scope
+    normalized_metadata_scope = (
+        validate_metadata_scope(raw_metadata_scope)
+        if raw_metadata_scope is not None
+        else None
+    )
+
     if "indexing" in data or "source" in data:
-        return DatasetConfig.from_dict(data, workdir=workdir)
+        return DatasetConfig.from_dict(
+            data,
+            workdir=workdir,
+            metadata_scope=normalized_metadata_scope,
+        )
 
     if "path" not in data:
         raise ValueError("Crawler config shorthand requires a 'path' field")
 
     ds_root = Path(_resolve_path(data["path"], base_path=workdir))
-    file_config = read_metadata_json(ds_root, CONFIG_FILENAME)
+    file_config = read_metadata_json(
+        ds_root,
+        CONFIG_FILENAME,
+        metadata_scope=normalized_metadata_scope,
+    )
     if file_config is None:
         raise FileNotFoundError(
             f"No {CONFIG_FILENAME} found at: {ds_root}"
+            + (
+                f" (metadata_scope={normalized_metadata_scope!r})"
+                if normalized_metadata_scope is not None
+                else ""
+            )
         )
-    return DatasetConfig.from_dict(file_config, dataset_root=ds_root)
+    return DatasetConfig.from_dict(
+        file_config,
+        dataset_root=ds_root,
+        metadata_scope=normalized_metadata_scope,
+    )
 
 
 @dataclass
@@ -412,6 +470,8 @@ class Config:
         cls,
         path: str | Path,
         workdir: str | Path | None = None,
+        *,
+        metadata_scope: str | None = None,
     ) -> "Config":
         path = Path(path)
         if not path.exists():
@@ -423,13 +483,21 @@ class Config:
         base_workdir = Path(workdir) if workdir is not None else path.parent
 
         if "datasets" not in data:
-            dataset = load_dataset_config(data, workdir=base_workdir)
+            dataset = load_dataset_config(
+                data,
+                workdir=base_workdir,
+                metadata_scope=metadata_scope,
+            )
             return cls(datasets=[dataset])
 
         datasets: list[DatasetConfig] = []
         for i, ds_data in enumerate(data["datasets"]):
             try:
-                ds_config = load_dataset_config(ds_data, workdir=base_workdir)
+                ds_config = load_dataset_config(
+                    ds_data,
+                    workdir=base_workdir,
+                    metadata_scope=metadata_scope,
+                )
                 datasets.append(ds_config)
             except FileNotFoundError:
                 raise