ds-crawler 2.13.0__py3-none-any.whl

ds_crawler/__init__.py

@@ -0,0 +1,119 @@
+"""Dataset crawler package."""
+
+from .artifact_builder import (
+    build_crawler_config,
+    build_dataset_artifacts_from_files,
+    build_dataset_head,
+)
+from .config import Config, DatasetConfig, load_dataset_config
+from .layout import (
+    EULER_LAYOUT_ADDON,
+    EULER_LAYOUT_VERSION,
+    build_layout_addon,
+    get_layout_addon,
+    validate_layout_addon,
+)
+from .migration import (
+    migrate_dataset_metadata,
+    migrate_dataset_zip,
+    migrate_dataset_zips_in_folder,
+)
+from .operations import (
+    HierarchySplitClause,
+    HierarchySplitRule,
+    align_datasets,
+    copy_dataset,
+    copy_dataset_splits,
+    create_aligned_dataset_splits,
+    create_dataset_splits,
+    create_hierarchy_dataset_splits,
+    create_mapped_dataset_splits,
+    extract_datasets,
+    list_dataset_splits,
+    load_dataset_split,
+    split_dataset,
+    split_datasets,
+)
+from .parser import (
+    DatasetParser,
+    index_dataset,
+    index_dataset_from_files,
+    index_dataset_from_path,
+)
+from .schema import (
+    DatasetDescriptor,
+    extract_dataset_properties,
+    get_dataset_contract,
+    get_dataset_properties,
+)
+from .traversal import (
+    collect_qualified_ids,
+    filter_index_by_qualified_ids,
+    get_files,
+    split_qualified_ids,
+)
+from .validation import validate_crawler_config, validate_dataset, validate_output
+from .writer import DatasetWriter, ZipDatasetWriter
+from .zip_utils import list_metadata_scopes, validate_metadata_scope
+
+__all__ = [
+    'ValidatedDatasetWriter',
+    'finalize_output',
+    'read_record',
+    'validate_output_records',
+
+    "DatasetDescriptor",
+    "DatasetWriter",
+    "ZipDatasetWriter",
+    "Config",
+    "DatasetConfig",
+    "DatasetParser",
+    "EULER_LAYOUT_ADDON",
+    "EULER_LAYOUT_VERSION",
+    "align_datasets",
+    "build_crawler_config",
+    "build_dataset_artifacts_from_files",
+    "build_dataset_head",
+    "build_layout_addon",
+    "collect_qualified_ids",
+    "copy_dataset",
+    "copy_dataset_splits",
+    "create_aligned_dataset_splits",
+    "create_dataset_splits",
+    "create_hierarchy_dataset_splits",
+    "create_mapped_dataset_splits",
+    "extract_datasets",
+    "HierarchySplitClause",
+    "HierarchySplitRule",
+    "filter_index_by_qualified_ids",
+    "get_files",
+    "get_dataset_contract",
+    "get_dataset_properties",
+    "get_layout_addon",
+    "index_dataset",
+    "index_dataset_from_files",
+    "index_dataset_from_path",
+    "list_dataset_splits",
+    "list_metadata_scopes",
+    "load_dataset_config",
+    "load_dataset_split",
+    "migrate_dataset_metadata",
+    "migrate_dataset_zip",
+    "migrate_dataset_zips_in_folder",
+    "split_dataset",
+    "split_datasets",
+    "split_qualified_ids",
+    "validate_crawler_config",
+    "validate_dataset",
+    "validate_metadata_scope",
+    "validate_layout_addon",
+    "validate_output",
+    "extract_dataset_properties",
+]
+
+from .records import (  # noqa: F401
+    ValidatedDatasetWriter,
+    finalize_output,
+    read_record,
+    validate_output_records,
+)

ds_crawler/_dataset_contract.py

@@ -0,0 +1,45 @@
+"""Imports from the shared ``euler-dataset-contract`` package."""
+
+from __future__ import annotations
+
+from euler_dataset_contract import (  # type: ignore[attr-defined]
+    DATASET_CONTRACT_VERSION,
+    DATASET_HEAD_KIND,
+    MODALITY_META_SCHEMAS,
+    DatasetHeadContract,
+    build_default_meta,
+    build_meta_schema,
+    get_registered_addon_validators,
+    normalize_meta_dict,
+    parse_dataset_head,
+    register_addon_validator,
+    validate_addon_version,
+    validate_contract_kind,
+    validate_contract_version,
+    validate_dataset_head,
+    validate_meta_dict,
+    validate_slot,
+    validate_string_list,
+    validate_token,
+)
+
+__all__ = [
+    "DATASET_CONTRACT_VERSION",
+    "DATASET_HEAD_KIND",
+    "MODALITY_META_SCHEMAS",
+    "DatasetHeadContract",
+    "build_default_meta",
+    "build_meta_schema",
+    "get_registered_addon_validators",
+    "normalize_meta_dict",
+    "parse_dataset_head",
+    "register_addon_validator",
+    "validate_addon_version",
+    "validate_contract_kind",
+    "validate_contract_version",
+    "validate_dataset_head",
+    "validate_meta_dict",
+    "validate_slot",
+    "validate_string_list",
+    "validate_token",
+]

ds_crawler/_version.py

@@ -0,0 +1,13 @@
+"""Package version helper."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+
+def get_package_version() -> str:
+    """Return the installed ds_crawler package version, or ``"0"`` as fallback."""
+    try:
+        return version("ds_crawler")
+    except PackageNotFoundError:
+        return "0"

ds_crawler/artifact_builder.py

@@ -0,0 +1,255 @@
+"""Public helpers for building canonical dataset artifacts."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Iterable
+
+from ._dataset_contract import (
+    DATASET_CONTRACT_VERSION,
+    build_default_meta,
+    parse_dataset_head,
+)
+from .artifacts import build_index_artifact
+from .config import (
+    CONFIG_FILENAME,
+    CRAWLER_CONFIG_KIND,
+    CRAWLER_CONFIG_VERSION,
+    DatasetConfig,
+)
+from .parser import index_dataset_from_files
+from .traversal import get_files
+from .zip_utils import (
+    DATASET_HEAD_FILENAME,
+    OUTPUT_FILENAME,
+    SCOPES_FILENAME,
+    _scopes_manifest_with,
+    get_metadata_filename,
+    validate_metadata_scope,
+)
+
+
+def _require_mapping(value: Any, context: str) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ValueError(f"{context} must be an object")
+    return value
+
+
+def _slugify(value: str) -> str:
+    cleaned = "".join(
+        ch.lower() if ch.isalnum() else "_"
+        for ch in value.strip()
+    )
+    cleaned = "_".join(token for token in cleaned.split("_") if token)
+    return cleaned or "dataset"
+
+
+def _reject_unknown_keys(
+    value: dict[str, Any],
+    *,
+    allowed: set[str],
+    context: str,
+) -> None:
+    unknown = sorted(set(value) - allowed)
+    if unknown:
+        joined = ", ".join(unknown)
+        raise ValueError(f"Unknown {context} key(s): {joined}")
+
+
+def _merge_meta_defaults(
+    modality_key: str,
+    raw_meta: Any,
+) -> dict[str, Any] | None:
+    defaults = build_default_meta(modality_key)
+    if raw_meta is None:
+        return defaults
+    if not isinstance(raw_meta, dict):
+        raise ValueError("modality.meta must be an object")
+    if defaults is None:
+        return dict(raw_meta)
+    merged = dict(defaults)
+    merged.update(raw_meta)
+    return merged
+
+
+def build_dataset_head(
+    *,
+    dataset: dict[str, Any],
+    modality: dict[str, Any],
+    addons: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build and validate a canonical ``dataset-head.json`` mapping."""
+    dataset = _require_mapping(dataset, "dataset")
+    modality = _require_mapping(modality, "modality")
+    addons = {} if addons is None else _require_mapping(addons, "addons")
+
+    _reject_unknown_keys(dataset, allowed={"id", "name", "attributes"}, context="dataset")
+    _reject_unknown_keys(modality, allowed={"key", "meta"}, context="modality")
+
+    dataset_name = dataset.get("name")
+    if not isinstance(dataset_name, str) or not dataset_name:
+        raise ValueError("dataset.name must be a non-empty string")
+
+    dataset_id = dataset.get("id")
+    if dataset_id is None:
+        dataset_id = _slugify(dataset_name)
+    elif not isinstance(dataset_id, str) or not dataset_id:
+        raise ValueError("dataset.id must be a non-empty string when provided")
+
+    attributes = dataset.get("attributes", {})
+    if attributes is None:
+        attributes = {}
+    if not isinstance(attributes, dict):
+        raise ValueError("dataset.attributes must be an object")
+
+    modality_key = modality.get("key")
+    if not isinstance(modality_key, str) or not modality_key:
+        raise ValueError("modality.key must be a non-empty string")
+
+    head = {
+        "contract": {
+            "kind": "dataset_head",
+            "version": DATASET_CONTRACT_VERSION,
+        },
+        "dataset": {
+            "id": dataset_id,
+            "name": dataset_name,
+        },
+        "modality": {
+            "key": modality_key,
+            "meta": _merge_meta_defaults(modality_key, modality.get("meta")),
+        },
+        "addons": deepcopy(addons),
+    }
+    if attributes:
+        head["dataset"]["attributes"] = deepcopy(attributes)
+    if not head["addons"]:
+        head.pop("addons")
+
+    return parse_dataset_head(head, context="dataset_head").to_mapping()
+
+
+def build_crawler_config(
+    *,
+    head: dict[str, Any],
+    indexing: dict[str, Any],
+    source_path: str = ".",
+    head_file: str = DATASET_HEAD_FILENAME,
+    prebuilt_index_file: str | None = None,
+    metadata_scope: str | None = None,
+) -> dict[str, Any]:
+    """Build and validate a canonical ``ds-crawler.json`` mapping."""
+    indexing = _require_mapping(indexing, "indexing")
+    if not isinstance(source_path, str) or not source_path:
+        raise ValueError("source_path must be a non-empty string")
+    if not isinstance(head_file, str) or not head_file:
+        raise ValueError("head_file must be a non-empty string")
+    if prebuilt_index_file is not None and (
+        not isinstance(prebuilt_index_file, str) or not prebuilt_index_file
+    ):
+        raise ValueError("prebuilt_index_file must be a non-empty string when provided")
+
+    config: dict[str, Any] = {
+        "contract": {
+            "kind": CRAWLER_CONFIG_KIND,
+            "version": CRAWLER_CONFIG_VERSION,
+        },
+        "head_file": head_file,
+        "source": {
+            "path": source_path,
+        },
+        "indexing": deepcopy(indexing),
+    }
+    if prebuilt_index_file is not None:
+        config["source"]["prebuilt_index_file"] = prebuilt_index_file
+    if metadata_scope is not None:
+        config["metadata_scope"] = validate_metadata_scope(metadata_scope)
+
+    runtime_config = dict(config)
+    runtime_config["head"] = deepcopy(head)
+    dataset_config = DatasetConfig.from_dict(
+        runtime_config,
+        dataset_root=Path(source_path),
+        dataset_head=head,
+    )
+    return dataset_config.to_crawler_mapping()
+
+
+def build_dataset_artifacts_from_files(
+    *,
+    dataset: dict[str, Any],
+    modality: dict[str, Any],
+    indexing: dict[str, Any],
+    files: Iterable[str | Path],
+    addons: dict[str, Any] | None = None,
+    base_path: str | Path | None = None,
+    strict: bool = False,
+    sample: int | None = None,
+    match_index: dict[str, Any] | None = None,
+    metadata_scope: str | None = None,
+) -> dict[str, Any]:
+    """Build canonical dataset artifacts from a parameter set and file list."""
+    normalized_scope = (
+        validate_metadata_scope(metadata_scope)
+        if metadata_scope is not None
+        else None
+    )
+    head = build_dataset_head(dataset=dataset, modality=modality, addons=addons)
+    config = build_crawler_config(
+        head=head,
+        indexing=indexing,
+        metadata_scope=normalized_scope,
+    )
+
+    runtime_config = dict(config)
+    runtime_config["head"] = deepcopy(head)
+    output = index_dataset_from_files(
+        runtime_config,
+        files,
+        base_path=base_path,
+        strict=strict,
+        sample=sample,
+        match_index=match_index,
+    )
+
+    artifacts = {
+        DATASET_HEAD_FILENAME: deepcopy(output["head"]),
+        CONFIG_FILENAME: config,
+        OUTPUT_FILENAME: build_index_artifact(output),
+    }
+    if normalized_scope is not None:
+        unscoped_artifact_names = list(artifacts)
+        artifacts = {
+            get_metadata_filename(
+                filename,
+                metadata_scope=normalized_scope,
+            ): payload
+            for filename, payload in artifacts.items()
+        }
+        artifacts[SCOPES_FILENAME] = _scopes_manifest_with(
+            None,
+            metadata_scope=normalized_scope,
+            filenames=unscoped_artifact_names,
+        )
+
+    summary = {
+        "dataset_id": output["head"]["dataset"]["id"],
+        "dataset_name": output["head"]["dataset"]["name"],
+        "modality_key": output["head"]["modality"]["key"],
+        "file_count": len(get_files(output)),
+    }
+    if normalized_scope is not None:
+        summary["metadata_scope"] = normalized_scope
+
+    return {
+        "artifacts": artifacts,
+        "summary": summary,
+    }
+
+
+__all__ = [
+    "build_crawler_config",
+    "build_dataset_artifacts_from_files",
+    "build_dataset_head",
+]

ds_crawler/artifacts.py

@@ -0,0 +1,247 @@
+"""Helpers for persisting and hydrating dataset metadata artifacts."""
+
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from pathlib import Path
+from typing import Any
+
+from .config import (
+    CONFIG_FILENAME,
+    CRAWLER_CONFIG_KIND,
+    CRAWLER_CONFIG_VERSION,
+    DatasetConfig,
+    load_dataset_config,
+)
+from .zip_utils import (
+    DATASET_HEAD_FILENAME,
+    METADATA_DIR,
+    OUTPUT_FILENAME,
+    is_zip_path,
+    read_metadata_json,
+    write_metadata_json_batch,
+)
+
+DATASET_SPLIT_KIND = "dataset_split"
+DATASET_SPLIT_VERSION = "1.0"
+
+
+def build_index_artifact(output: dict[str, Any]) -> dict[str, Any]:
+    """Return the minimal on-disk index artifact for a full dataset output."""
+    if not isinstance(output, dict):
+        raise ValueError("output must be an object")
+    if "contract" not in output or "index" not in output:
+        raise ValueError("output must contain contract and index")
+
+    artifact = {
+        "contract": output["contract"],
+        "index": output["index"],
+    }
+    generator = output.get("generator")
+    if generator is not None:
+        artifact["generator"] = generator
+    execution = output.get("execution")
+    if execution is not None:
+        artifact["execution"] = execution
+    return artifact
+
+
+def build_split_artifact(
+    output: dict[str, Any],
+    *,
+    split_name: str,
+    source_index_file: str = OUTPUT_FILENAME,
+    execution: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Return the on-disk artifact for an inline split."""
+    if not isinstance(output, dict):
+        raise ValueError("output must be an object")
+    if "index" not in output or not isinstance(output["index"], dict):
+        raise ValueError("output.index must be an object")
+    if not isinstance(split_name, str) or not split_name:
+        raise ValueError("split_name must be a non-empty string")
+    if not isinstance(source_index_file, str) or not source_index_file:
+        raise ValueError("source_index_file must be a non-empty string")
+
+    artifact = {
+        "contract": {
+            "kind": DATASET_SPLIT_KIND,
+            "version": DATASET_SPLIT_VERSION,
+        },
+        "split": {
+            "name": split_name,
+            "source_index_file": source_index_file,
+        },
+        "index": deepcopy(output["index"]),
+    }
+    generator = output.get("generator")
+    if generator is not None:
+        artifact["generator"] = deepcopy(generator)
+    if execution:
+        artifact["execution"] = deepcopy(execution)
+    return artifact
+
+
+def build_crawler_config_for_output(output: dict[str, Any]) -> dict[str, Any]:
+    """Derive a crawler config mapping from a full dataset output."""
+    if not isinstance(output, dict):
+        raise ValueError("output must be an object")
+    if "head" not in output or not isinstance(output["head"], dict):
+        raise ValueError("output.head must be an object")
+
+    head_file = output.get("head_file", DATASET_HEAD_FILENAME)
+    if not isinstance(head_file, str) or not head_file:
+        raise ValueError("output.head_file must be a non-empty string")
+
+    indexing = output.get("indexing", {})
+    if not isinstance(indexing, dict):
+        raise ValueError("output.indexing must be an object")
+
+    config = {
+        "contract": {
+            "kind": CRAWLER_CONFIG_KIND,
+            "version": CRAWLER_CONFIG_VERSION,
+        },
+        "head_file": head_file,
+        "source": {
+            "path": ".",
+        },
+        "indexing": dict(indexing),
+    }
+
+    id_cfg = indexing.get("id")
+    has_id_regex = (
+        isinstance(id_cfg, dict)
+        and isinstance(id_cfg.get("regex"), str)
+        and bool(id_cfg.get("regex"))
+    )
+    if not has_id_regex:
+        config["source"]["prebuilt_index_file"] = OUTPUT_FILENAME
+
+    return config
+
+
+def hydrate_index_artifact(
+    index_artifact: dict[str, Any],
+    ds_config: DatasetConfig,
+) -> dict[str, Any]:
+    """Expand an on-disk index artifact into the richer in-memory output shape."""
+    if not isinstance(index_artifact, dict):
+        raise ValueError("index_artifact must be an object")
+    if "contract" not in index_artifact or "index" not in index_artifact:
+        raise ValueError("index_artifact must contain contract and index")
+
+    hydrated = dict(index_artifact)
+    hydrated["head_file"] = ds_config.head_file
+    hydrated["head"] = ds_config.head_mapping()
+    hydrated["indexing"] = ds_config.to_indexing_dict()
+    hydrated.setdefault("generator", {})
+    hydrated.setdefault("execution", {})
+    return hydrated
+
+
+def hydrate_split_artifact(
+    split_artifact: dict[str, Any],
+    base_output: dict[str, Any],
+) -> dict[str, Any]:
+    """Expand a split artifact into the richer in-memory output shape."""
+    if not isinstance(split_artifact, dict):
+        raise ValueError("split_artifact must be an object")
+    if "index" not in split_artifact or "split" not in split_artifact:
+        raise ValueError("split_artifact must contain split and index")
+    if not isinstance(base_output, dict):
+        raise ValueError("base_output must be an object")
+
+    hydrated = dict(base_output)
+    hydrated["index"] = deepcopy(split_artifact["index"])
+    hydrated["split"] = deepcopy(split_artifact["split"])
+
+    split_execution = split_artifact.get("execution")
+    if isinstance(split_execution, dict) and split_execution:
+        execution = dict(hydrated.get("execution") or {})
+        execution["split"] = deepcopy(split_execution)
+        hydrated["execution"] = execution
+
+    generator = split_artifact.get("generator")
+    if "generator" not in hydrated and isinstance(generator, dict):
+        hydrated["generator"] = deepcopy(generator)
+    return hydrated
+
+
+def load_saved_output(
+    dataset_path: str | Path,
+    *,
+    filename: str = OUTPUT_FILENAME,
+    metadata_scope: str | None = None,
+) -> dict[str, Any] | None:
+    """Load and hydrate a persisted dataset index artifact from disk."""
+    dataset_root = Path(dataset_path)
+    index_artifact = read_metadata_json(
+        dataset_root,
+        filename,
+        metadata_scope=metadata_scope,
+    )
+    if index_artifact is None:
+        return None
+    ds_config = load_dataset_config(
+        {"path": str(dataset_root)},
+        metadata_scope=metadata_scope,
+    )
+    return hydrate_index_artifact(index_artifact, ds_config)
+
+
+def load_prebuilt_output(ds_config: DatasetConfig) -> dict[str, Any]:
+    """Load and hydrate a prebuilt index configured by ``prebuilt_index_file``."""
+    if ds_config.prebuilt_index_file is None:
+        raise FileNotFoundError("Dataset config has no prebuilt index file")
+
+    prebuilt_path = Path(ds_config.prebuilt_index_file)
+    if prebuilt_path.is_file():
+        with open(prebuilt_path) as f:
+            index_artifact = json.load(f)
+    else:
+        index_artifact = read_metadata_json(
+            Path(ds_config.dataset_root),
+            prebuilt_path.name,
+            metadata_scope=ds_config.metadata_scope,
+        )
+        if index_artifact is None:
+            raise FileNotFoundError(
+                f"No prebuilt index found at {ds_config.prebuilt_index_file}"
+            )
+
+    return hydrate_index_artifact(index_artifact, ds_config)
+
+
+def save_output_artifacts(
+    dataset_path: str | Path,
+    output: dict[str, Any],
+    *,
+    filename: str = OUTPUT_FILENAME,
+    metadata_scope: str | None = None,
+) -> Path:
+    """Persist head/config/index artifacts for a dataset output."""
+    dataset_root = Path(dataset_path)
+    if "head" not in output or not isinstance(output["head"], dict):
+        raise ValueError("output.head must be an object")
+
+    head_file = output.get("head_file", DATASET_HEAD_FILENAME)
+    if not isinstance(head_file, str) or not head_file:
+        raise ValueError("output.head_file must be a non-empty string")
+
+    write_metadata_json_batch(
+        dataset_root,
+        {
+            head_file: output["head"],
+            CONFIG_FILENAME: build_crawler_config_for_output(output),
+            filename: build_index_artifact(output),
+        },
+        metadata_scope=metadata_scope,
+    )
+
+    if is_zip_path(dataset_root):
+        return dataset_root
+    if metadata_scope is not None:
+        return dataset_root / METADATA_DIR / metadata_scope / filename
+    return dataset_root / METADATA_DIR / filename