plexus-python-pulse-demeter 1.1.126__py3-none-any.whl

plexus/pulse/demeter/utils/datautils.py

@@ -0,0 +1,436 @@
+import dataclasses
+import fnmatch
+import json
+import re
+import uuid as py_uuid
+from pathlib import Path
+from typing import Iterator
+
+from cloudpathlib import AnyPath, CloudPath
+from plexus.common.utils.funcutils import singleton
+from returns.result import Failure, Result, Success
+
+from plexus.pulse.demeter.app.models.local import DatasetMetafile, SampleMetafile, StandaloneDataset, StandaloneSample
+from plexus.pulse.demeter.app.models.registry.dataset import SampleLayout
+
+__all__ = [
+    "dataset_metafile_filename",
+    "sample_metafile_filename",
+    "walk_standalone_sample",
+    "validate_standalone_sample",
+    "collect_standalone_dataset_samples",
+]
+
+
+@singleton
+def dataset_metafile_filename() -> str:
+    return "pulse-demeter-dataset.json"
+
+
+@singleton
+def sample_metafile_filename() -> str:
+    return "pulse-demeter-sample.json"
+
+
+@dataclasses.dataclass(frozen=True)
+class ValidationFailure(object):
+    path: str
+    reason: str
+    exception: Exception | None = None
+
+
+def find_layout(
+    layout_reference: str | py_uuid.UUID,
+    layouts: list[SampleLayout],
+) -> SampleLayout | None:
+    for layout in layouts:
+        if isinstance(layout_reference, py_uuid.UUID):
+            if layout.uuid == layout_reference:
+                return layout
+        else:
+            if layout.name == layout_reference:
+                return layout
+    return None
+
+
+def retrieve_dataset_metafile(
+    dataset_path: Path | CloudPath,
+) -> Result[DatasetMetafile | None, ValidationFailure]:
+    """
+    Retrieve the dataset metafile for a given dataset path.
+
+    :param dataset_path: absolute path to the dataset root
+
+    :return: Result containing optional ``DatasetMetafile``, or a ``ValidationFailure`` on error.
+    """
+    dataset_metafile_path = dataset_path / dataset_metafile_filename()
+
+    if dataset_metafile_path.is_file():
+        try:
+            with dataset_metafile_path.open("r", encoding="utf-8") as fh:
+                metafile = DatasetMetafile.model_validate(json.load(fh))
+            return Success(metafile)
+        except Exception as e:
+            return Failure(
+                ValidationFailure("", f"error reading and parsing dataset metafile '{dataset_metafile_filename()}'", e))
+    else:
+        # no dataset metafile found, considered as non-fatal
+        return Success(None)
+
+
+def retrieve_sample_metafile_and_layout(
+    sample_path: Path | CloudPath,
+    layouts: list[SampleLayout],
+) -> Result[tuple[SampleMetafile | None, SampleLayout | None], ValidationFailure]:
+    """
+    Retrieve the sample metafile and corresponding layout for a given sample path.
+
+    :param sample_path: absolute path to the sample root
+    :param layouts: list of available ``SampleLayout`` objects
+
+    :return: Result containing optional tuple of ``SampleMetafile`` and ``SampleLayout``, or a ``ValidationFailure``
+             on error.
+    """
+    sample_metafile_path = sample_path / sample_metafile_filename()
+
+    if sample_metafile_path.is_file():
+        try:
+            with sample_metafile_path.open("r", encoding="utf-8") as fh:
+                metafile = SampleMetafile.model_validate(json.load(fh))
+        except Exception as e:
+            return Failure(
+                ValidationFailure("", f"error reading and parsing sample metafile '{sample_metafile_filename()}'", e))
+    else:
+        # no sample metafile found, considered as non-fatal
+        return Success((None, None))
+
+    layout_reference = metafile.layout_uuid or metafile.layout_name
+    if layout_reference is None:
+        return Failure(ValidationFailure("", f"missing 'layout_uuid' or 'layout_name' in sample metafile"))
+
+    layout = find_layout(layout_reference, layouts)
+    if layout is None:
+        return Failure(ValidationFailure("", f"layout of reference '{layout_reference}' is missing"))
+
+    return Success((metafile, layout))
+
+
+@singleton
+def anonymous_layout_uuid() -> py_uuid.UUID:
+    """UUID used for anonymous layouts in sample validation."""
+    return py_uuid.UUID("00000000-0000-0000-0000-000000000000")
+
+
+def walk_standalone_sample(
+    dataset_path: Path | CloudPath,
+    sample_path: Path | CloudPath,
+    layout: SampleLayout,
+    layouts: list[SampleLayout],
+    *,
+    ignore_metafile: bool = False,
+) -> Iterator[Result[StandaloneSample, ValidationFailure]]:
+    """
+    Walk and validate a standalone sample against its layout.
+
+    :param dataset_path: absolute path to the dataset root
+    :param sample_path: absolute path to the sample root
+    :param layout: layout to validate against
+    :param layouts: list of available ``SampleLayout`` objects
+    :param ignore_metafile: whether to ignore the sample metafile during validation
+
+    :return: Iterator of Results containing ``StandaloneSample`` or ``ValidationFailure``
+    """
+    if not dataset_path.is_absolute():
+        raise ValueError(f"'{dataset_path}' is not an absolute path")
+    if not sample_path.is_absolute():
+        raise ValueError(f"'{sample_path}' is not an absolute path")
+
+    relpath = str(sample_path.relative_to(dataset_path))
+
+    if not sample_path.exists():
+        yield Failure(ValidationFailure(relpath, f"missing required path '{relpath}'"))
+        return
+
+    if layout is None:
+        yield Failure(ValidationFailure(relpath, "missing layout for sample validation"))
+        return
+
+    metafile = None
+    metafile_layout = None
+    if not ignore_metafile:
+        match retrieve_sample_metafile_and_layout(sample_path, layouts):
+            case Success((metafile, metafile_layout)):
+                pass
+            case Failure(failure):
+                yield Failure(ValidationFailure(relpath, failure.reason, failure.exception))
+                return
+
+    if not ignore_metafile:
+        if layout.uuid != anonymous_layout_uuid():
+            if metafile_layout is None:
+                yield Failure(ValidationFailure(relpath, "missing sample metafile for layout validation"))
+                return
+            elif layout.uuid != metafile_layout.uuid:
+                yield Failure(ValidationFailure(relpath,
+                                                f"layout UUID '{layout.uuid}' does not match sample meta layout UUID '{metafile_layout.uuid}'"))
+                return
+
+    layout_spec = layout.layout_spec.get("layout")
+    layout_spec_type = layout_spec.get("type")
+
+    if layout_spec_type == "uuid":
+        # If the layout spec references a layout by UUID, validate against the layout specified in the sample metafile
+        try:
+            layout = find_layout(py_uuid.UUID(layout_spec.get("value")), layouts)
+        except ValueError as e:
+            yield Failure(ValidationFailure(relpath, "invalid UUID in layout spec", e))
+            return
+
+        yield from walk_standalone_sample(dataset_path,
+                                          sample_path,
+                                          layout,
+                                          layouts,
+                                          ignore_metafile=ignore_metafile)
+        return
+
+    if layout_spec_type == "name":
+        # If the layout spec references a layout by name, validate against the layout specified in the sample metafile
+        layout = find_layout(layout_spec.get("value"), layouts)
+
+        yield from walk_standalone_sample(dataset_path,
+                                          sample_path,
+                                          layout,
+                                          layouts,
+                                          ignore_metafile=ignore_metafile)
+        return
+
+    if layout_spec_type == "file" and not sample_path.is_file():
+        yield Failure(ValidationFailure(relpath, "expected file but not a file"))
+        return
+
+    if layout_spec_type == "directory" and not sample_path.is_dir():
+        yield Failure(ValidationFailure(relpath, "expected directory but not a directory"))
+        return
+
+    if layout_spec_type == "file":
+        return
+
+    layout_spec_children = layout_spec.get("children", [])
+    allow_extra = layout_spec.get("allow_extra", True)
+
+    try:
+        entries = sorted(list(sample_path.iterdir()))
+    except NotADirectoryError as e:
+        yield Failure(ValidationFailure(relpath, "expected directory to list children", e))
+        return
+
+    child_samples: list[StandaloneSample] = []
+    matched_entries: set[Path | CloudPath] = set()
+
+    for idx, layout_spec_child in enumerate(layout_spec_children):
+        child_matcher = layout_spec_child.get("matcher")
+        if child_matcher is None:
+            yield Failure(ValidationFailure(relpath, f"child spec at index {idx} missing 'matcher' key"))
+            continue
+
+        child_layout = SampleLayout(uuid=anonymous_layout_uuid(),
+                                    name="anonymous",
+                                    description="",
+                                    layout_spec={"layout": layout_spec_child.get("layout")},
+                                    tags=[],
+                                    props={},
+                                    )
+
+        child_matcher_type = child_matcher.get("type")
+        child_matcher_pattern = child_matcher.get("pattern")
+
+        if child_matcher_type == "exact":
+            def matcher_func(p: Path | CloudPath) -> bool:
+                return p.name == child_matcher_pattern
+        elif child_matcher_type == "regex":
+            def matcher_func(p: Path | CloudPath) -> bool:
+                return re.match(child_matcher_pattern, p.name) is not None
+        elif child_matcher_type == "glob":
+            def matcher_func(p: Path | CloudPath) -> bool:
+                return fnmatch.fnmatch(p.name, child_matcher_pattern)
+        else:
+            def matcher_func(p: Path | CloudPath) -> bool:
+                return True  # match all if no type specified
+
+        matches_count = 0
+        for entry in entries:
+            if not matcher_func(entry):
+                continue
+
+            if entry in matched_entries:
+                yield Failure(ValidationFailure(relpath, f"entry '{entry.name}' matched multiple times"))
+
+            matched_entries.add(entry)
+            matches_count += 1
+
+            for result in walk_standalone_sample(dataset_path,
+                                                 entry,
+                                                 child_layout,
+                                                 layouts,
+                                                 ignore_metafile=ignore_metafile):
+                match result:
+                    case Success(sample):
+                        child_samples.append(sample)
+                    case _:
+                        pass
+                yield result
+
+        min_items = child_matcher.get("min_items")
+        if min_items is not None and matches_count < min_items:
+            yield Failure(ValidationFailure(relpath,
+                                            f"found {matches_count} matches for '{child_matcher_pattern}' < min_items {min_items}"))
+
+        max_items = child_matcher.get("max_items")
+        if max_items is not None and matches_count > max_items:
+            yield Failure(ValidationFailure(relpath,
+                                            f"found {matches_count} matches for '{child_matcher_pattern}' > max_items {max_items}"))
+
+    if not allow_extra:
+        for entry in entries:
+            if entry in matched_entries:
+                continue
+            if entry.is_file() and entry.name == sample_metafile_filename():
+                continue  # allow sample metafile by default
+            yield Failure(ValidationFailure(relpath, f"unexpected extra entry '{entry.name}'"))
+
+    if metafile is not None:
+        try:
+            sample = StandaloneSample(path=relpath,
+                                      layout_uuid=layout.uuid,
+                                      bag_name=metafile.bag_name,
+                                      vehicle_name=metafile.vehicle_name,
+                                      begin_frame_dt=metafile.begin_frame_dt,
+                                      end_frame_dt=metafile.end_frame_dt,
+                                      child_sample_paths=[child_sample.path for child_sample in child_samples],
+                                      tags=metafile.tags,
+                                      props=metafile.props,
+                                      )
+            yield Success(sample)
+        except Exception as e:
+            yield Failure(ValidationFailure(relpath, "error constructing 'StandaloneSample'", e))
+
+
+def validate_standalone_sample(
+    dataset_root: str | Path | CloudPath,
+    sample_root: str | Path | CloudPath,
+    layout_reference: str | py_uuid.UUID,
+    layouts: list[SampleLayout],
+) -> list[ValidationFailure]:
+    """
+    Validate a standalone sample against its layout.
+
+    :param dataset_root: absolute path to the dataset root
+    :param sample_root: absolute or relative path to the sample root
+    :param layout_reference: layout reference (name or UUID) to validate against
+    :param layouts: list of available ``SampleLayout`` objects
+
+    :return: list of ``ValidationFailure`` objects encountered during validation
+    """
+    dataset_root_path = AnyPath(dataset_root)
+    if not dataset_root_path.is_absolute():
+        raise ValueError(f"'{dataset_root_path}' is not an absolute path")
+
+    sample_root_path = AnyPath(sample_root)
+    if not sample_root_path.is_absolute():
+        sample_root_path = dataset_root_path / sample_root_path
+
+    relpath = str(sample_root_path.relative_to(dataset_root_path))
+
+    referred_layout = find_layout(layout_reference, layouts)
+    if referred_layout is None:
+        return [ValidationFailure(relpath, f"layout of reference '{layout_reference}' is missing")]
+
+    failures: list[ValidationFailure] = []
+    for result in walk_standalone_sample(dataset_root_path,
+                                         sample_root_path,
+                                         referred_layout,
+                                         layouts,
+                                         ignore_metafile=True):
+        match result:
+            case Success(_):
+                pass
+            case Failure(failure):
+                failures.append(failure)
+
+    return failures
+
+
+def collect_standalone_dataset_samples(
+    dataset_root: str | Path | CloudPath,
+    layouts: list[SampleLayout],
+) -> tuple[StandaloneDataset | None, list[StandaloneSample], list[ValidationFailure]]:
+    """
+    Collect standalone samples from a dataset root.
+
+    :param dataset_root: absolute path to the dataset root
+    :param layouts: list of available ``SampleLayout`` objects
+
+    :return: tuple containing optional ``StandaloneDataset``, list of ``StandaloneSample``, and list of
+             ``ValidationFailure`` objects
+    """
+    dataset_root_path = AnyPath(dataset_root)
+    if not dataset_root_path.is_absolute():
+        raise ValueError(f"'{dataset_root_path}' is not an absolute path")
+
+    metafile = None
+    match retrieve_dataset_metafile(dataset_root_path):
+        case Success(metafile):
+            pass
+        case Failure(failure):
+            return None, [], [failure]
+
+    if metafile is None:
+        return None, [], [ValidationFailure("", f"missing dataset metafile in '{dataset_root_path}'")]
+    try:
+        dataset = StandaloneDataset(name=metafile.name,
+                                    description=metafile.description,
+                                    contributor=metafile.contributor,
+                                    tags=metafile.tags,
+                                    props=metafile.props,
+                                    )
+    except Exception as e:
+        return None, [], [ValidationFailure("", f"error constructing 'StandaloneDataset'", e)]
+
+    samples: list[StandaloneSample] = []
+    failures: list[ValidationFailure] = []
+
+    relpath_dedupe: set[str] = set()
+    for sample_path in dataset_root_path.rglob("*"):
+        if not sample_path.is_dir():
+            continue
+
+        relpath = str(sample_path.relative_to(dataset_root_path))
+
+        if relpath in relpath_dedupe:
+            continue
+
+        match retrieve_sample_metafile_and_layout(sample_path, layouts):
+            case Failure(failure):
+                failures.append(ValidationFailure(relpath, failure.reason, failure.exception))
+                continue
+
+            case Success((_, metafile_layout)):
+                if metafile_layout is None:
+                    continue
+
+                sample_relpath_dedupe: set[str] = set()
+
+                for result in walk_standalone_sample(dataset_root_path, sample_path, metafile_layout, layouts):
+                    match result:
+                        case Success(sample):
+                            if sample.path not in relpath_dedupe:
+                                sample_relpath_dedupe.add(sample.path)
+                                samples.append(sample)
+                        case Failure(failure):
+                            if failure.path not in relpath_dedupe:
+                                sample_relpath_dedupe.add(failure.path)
+                                failures.append(failure)
+
+                relpath_dedupe.update(sample_relpath_dedupe)
+
+    return dataset, samples, failures

plexus_python_pulse_demeter-1.1.126.dist-info/METADATA

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: plexus-python-pulse-demeter
+Version: 1.1.126
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: <3.14,>=3.12
+Requires-Dist: cloudpathlib>=0.21
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: pydantic-settings>=2.12
+Requires-Dist: returns>=0.26
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: sqlmodel
+Requires-Dist: textcase
+Requires-Dist: plexus-python-common[all]>=1.1
+Requires-Dist: plexus-python-pulse-common[all]>=1.1
+Provides-Extra: all
+Requires-Dist: plexus-python-pulse-demeter[api,cli]; extra == "all"
+Provides-Extra: api
+Requires-Dist: alembic; extra == "api"
+Requires-Dist: fastapi; extra == "api"
+Requires-Dist: httpx; extra == "api"
+Requires-Dist: psycopg; extra == "api"
+Requires-Dist: python-multipart; extra == "api"
+Requires-Dist: ujson; extra == "api"
+Requires-Dist: uvicorn; extra == "api"
+Provides-Extra: cli
+Requires-Dist: argcomplete; extra == "cli"
+Provides-Extra: test
+Requires-Dist: ddt>=1.7; extra == "test"
+Requires-Dist: moto[s3,server]>=5.1; extra == "test"
+Requires-Dist: pytest-cov>=5.0; extra == "test"
+Requires-Dist: pytest-httpx>=0.35; extra == "test"
+Requires-Dist: pytest-order>=1.3; extra == "test"
+Requires-Dist: pytest-postgresql>=6.1; extra == "test"
+Requires-Dist: pytest>=8.3; extra == "test"

plexus_python_pulse_demeter-1.1.126.dist-info/RECORD

@@ -0,0 +1,31 @@
+plexus/pulse/demeter/__init__.py,sha256=vxHFsMrvJXWVyXVYFsDiQ_IUElGstGTasjWpGAwWHzA,184
+plexus/pulse/demeter/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plexus/pulse/demeter/app/main.py,sha256=bd67HvdGV71zDxzJJkViUxTq4IhesJ18VfQUV2jzAws,325
+plexus/pulse/demeter/app/api/__init__.py,sha256=0TeS_lTsRpOGJH-ubKiE2V4pd-rIrCU1e25qvoUKSEs,1637
+plexus/pulse/demeter/app/api/archive_internal_endpoints.py,sha256=Qzhg16ZPdtgCehF1ilUHOkcgS_DskzL7bBkIivz1fSU,11750
+plexus/pulse/demeter/app/api/archive_public_endpoints.py,sha256=uyoGCtVeGpzOJpnbdv0oFa-borp9YNLH-tUx-gVPmuY,9295
+plexus/pulse/demeter/app/api/dataset_internal_endpoints.py,sha256=oNS61daRi3sa2yt_vGAkmZ22CY3Tvam6kpOygiLOKK0,40321
+plexus/pulse/demeter/app/api/dataset_public_endpoints.py,sha256=ghhyOMh25scwQmJB3_G5RKSGMTZbJgaUTPCk3v03ysE,98645
+plexus/pulse/demeter/app/api/dataset_workflow_endpoints.py,sha256=sYdDVy6Vi5NeSjczEpXs-fa3I3JxOenApBSB6t6F20k,23028
+plexus/pulse/demeter/app/api/tagging_endpoints.py,sha256=ur56K0jNuMn2CNn97mC7cPA7HqFWQP7IHKeRvHqQfK4,12652
+plexus/pulse/demeter/app/api/user_endpoints.py,sha256=zhm407bH7rJBidBRRYSfmMUNaF7I8-beGADA_AlLlFA,2716
+plexus/pulse/demeter/app/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plexus/pulse/demeter/app/core/database.py,sha256=gzZTxNG2Jyhvceem4E5ea1ttN_f9ldSugPEYFvTh7WE,1296
+plexus/pulse/demeter/app/core/vault.py,sha256=zVVXwqk8Wva_S7RsgIAp7a1hz9xSiGGd5kwYiDzV3GA,1073
+plexus/pulse/demeter/app/models/__init__.py,sha256=vTaUkrM74ECgdzXY5BhN9RLm4jpBgrmBK7vdckS0MrM,750
+plexus/pulse/demeter/app/models/local.py,sha256=ApmHBD5RYgtoiwKw8oWXOPsmk7o9ifPm8IbQhT1zjxY,7641
+plexus/pulse/demeter/app/models/registry/__init__.py,sha256=HV1Gf27ARIbD6ix7Ebif_3kDg-65mPpjxDWAStdZ9ac,2923
+plexus/pulse/demeter/app/models/registry/archive.py,sha256=iU1niJQHfFKeoQgu-6Sx_30YlfP5K5EA0uL7LLvS4MA,19588
+plexus/pulse/demeter/app/models/registry/dataset.py,sha256=5u7aJ3HmgDG6lsiHhQtIzvpjwGEn-XZkH4ip46Qn-7E,66679
+plexus/pulse/demeter/app/models/registry/tagging.py,sha256=BZD7xql9uzTUwBJLKndENDNHvilzm8mCAOlNGNrQx_0,18364
+plexus/pulse/demeter/app/models/registry/user.py,sha256=lDE2H8kzZ5Kwp2QbAOxj4A-29ybX7A5xTek6MzEHnu4,2532
+plexus/pulse/demeter/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plexus/pulse/demeter/tools/main.py,sha256=SOL6pYvHciYl698kmZtdr_lYue7J_8U4ILP6ZaxwTpg,12142
+plexus/pulse/demeter/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+plexus/pulse/demeter/utils/config.py,sha256=Ij9wkGpSGIN2NwQzeB3eewftZrdMsAJQfBjp2PuFARs,947
+plexus/pulse/demeter/utils/datautils.py,sha256=gOUsii1ZrkStjAOxcmN7tuhhTz3dGZdizo23VJTVAAo,17250
+plexus_python_pulse_demeter-1.1.126.dist-info/METADATA,sha256=-8P8pzIP8ZrDl-WJ_P__wrBVc9xGQPbiceo23HeSSas,1375
+plexus_python_pulse_demeter-1.1.126.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+plexus_python_pulse_demeter-1.1.126.dist-info/entry_points.txt,sha256=XGj43U0GF9wwHdrVvsUXNiwF0su7RVjCJgC2kD2saSY,78
+plexus_python_pulse_demeter-1.1.126.dist-info/top_level.txt,sha256=ug_g7CVwaMQuas5UzAXbHUrQvKGCn8ezc6ZNvvRlJOE,7
+plexus_python_pulse_demeter-1.1.126.dist-info/RECORD,,

plexus_python_pulse_demeter-1.1.126.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

plexus_python_pulse_demeter-1.1.126.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+plexus-demeter-tools = plexus.pulse.demeter.tools.main:main

plexus_python_pulse_demeter-1.1.126.dist-info/top_level.txt

@@ -0,0 +1 @@
+plexus