ialdev-dataman 0.2.3__tar.gz → 0.2.5__tar.gz

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ialdev-dataman
-Version: 0.2.3
+Version: 0.2.5
 Summary: Imported as `iad.dataman` — data casting, resource models, and dataset bridge
 Author: ipcoder
 Requires-Python: >=3.10
@@ -12,12 +12,12 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: ialdev-core>=0.2.3
-Requires-Dist: ialdev-io>=0.2.1
-Requires-Dist: ialdev-img>=0.2.1
-Requires-Dist: ialdev-maths>=0.2.2
+Requires-Dist: ialdev-core>=0.2.4
+Requires-Dist: ialdev-io>=0.2.2
+Requires-Dist: ialdev-img>=0.2.2
+Requires-Dist: ialdev-maths>=0.2.3
 Requires-Dist: pandas>=2.0.0
-Requires-Dist: numpy>=1.26.4
+Requires-Dist: numpy>=1.26.4,<2
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=5.4.0
 Requires-Dist: regex>=2021.0.0
@@ -41,7 +41,7 @@ Use this package to describe datasets with resource models, discover YAML-backed
 pip install ialdev-dataman
 ```
 
-Requires Python `>=3.10`, NumPy `>=2.0`, and pandas `>=2.0.0`.
+Requires Python `>=3.10`, NumPy `>=1.26.4,<2`, and pandas `>=2.0.0`.
 
 ## Highlights
 

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/README.md

@@ -10,7 +10,7 @@ Use this package to describe datasets with resource models, discover YAML-backed
 pip install ialdev-dataman
 ```
 
-Requires Python `>=3.10`, NumPy `>=2.0`, and pandas `>=2.0.0`.
+Requires Python `>=3.10`, NumPy `>=1.26.4,<2`, and pandas `>=2.0.0`.
 
 ## Highlights
 

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "ialdev-dataman"
-version = "0.2.3"
+version = "0.2.5"
 description = "Imported as `iad.dataman` — data casting, resource models, and dataset bridge"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -23,12 +23,12 @@ classifiers = [
 ]
 
 dependencies = [
-    "ialdev-core>=0.2.3",
-    "ialdev-io>=0.2.1",
-    "ialdev-img>=0.2.1",
-    "ialdev-maths>=0.2.2",
+    "ialdev-core>=0.2.4",
+    "ialdev-io>=0.2.2",
+    "ialdev-img>=0.2.2",
+    "ialdev-maths>=0.2.3",
     "pandas>=2.0.0",
-    "numpy>=1.26.4",
+    "numpy>=1.26.4,<2",
     "pydantic>=2.0",
     "pyyaml>=5.4.0",
     "regex>=2021.0.0",

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/src/iad/dataman/__init__.py

@@ -4,7 +4,7 @@ from .models import DataSourceRM, SchemeRM, DatasetRM, CollectionRM, DSample, di
 from .factories import create_caster, create_collection, create_sink
 from .datacast.collect import DataCollection, SinkRepo
 
-__version__ = "0.2.3"
+__version__ = "0.2.5"
 
 __all__ = [
     "__version__",

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/src/iad/dataman/datacast/caster.py

@@ -32,13 +32,13 @@ class CasterConfig:
     __slots__ = ('name', 'root', 'search', 'labels', 'mappings',
                  'reverse', 'bundle', 'filters', 'sample')
 
-    def __init__(self, *, name: str = None, root: Path | str,
+    def __init__(self, *, name: str = None, root: Path | str | None,
                  search: GuideScan, labels: dict = None,
                  mappings: dict = None, reverse: bool = False,
                  bundle: list | None = None, filters: dict = None,
                  sample: dict = None):
         self.name = name
-        self.root = Path(root) if root else None
+        self.root = root and Path(root)
         self.search = search
         self.labels = labels or {}
         self.mappings = mappings or {}
@@ -116,7 +116,7 @@ class DataCaster:
     CacheMode = CacheMode  # allow PathScheme.CacheMode instead of scheme.CacheMode
 
     def __init__(self, name: str = None, *,
-                 root: Path | str,
+                 root: Path | str | list[str],
                  search: GuideScan,
                  labels: dict = None,
                  mappings: dict = None,
@@ -128,16 +128,24 @@ class DataCaster:
                  temp_cache=False, cache: Optional[CacheMode | bool] = True,
                  progress: dict | bool = None):
         """
-        Create data caster semantic parser for particularly structured files tree.
+        Create a data caster semantic parser for a particularly structured files tree.
         ::
             tree -> [pattern] -> files -> [label] -> [filter] -> labeled
 
         All parameters must be fully resolved (no name-based lookup).
-        For name-based construction (e.g. ``DataCaster('ETH3D')``), use the
-        factory in ``dataman`` (e.g. ``dataman.create_caster``).
+        For name-based construction (e.g. `DataCaster('ETH3D')`), use the
+        factory in `dataman` (e.g. ``dataman.create_caster``).
+
+        **Debug Mode**
+        There is a special "debug" mode, when instead of a root of the tree, a list of paths is provided.
+        That allows evaluating the labeling pipeline on a specific set of files
+        without the need for a full tree structure.
+        (`GuideScan.samples` may be used as such input if the scheme defines them)
+        In this case caching is always disabled.
         """
         self.config = CasterConfig(
-            name=name, root=root, search=search, labels=labels,
+            name=name, root=root if isinstance(root, (Path, str)) else '',
+            search=search, labels=labels,
             mappings=mappings, reverse=reverse, bundle=bundle,
             filters=filters, sample=sample,
         )
@@ -146,6 +154,11 @@ class DataCaster:
                           reverse=self.config.reverse, namespace=self.labeling_namespace)
         self.categories = labeler.categories
         self.bundle = self.config.bundle or []
+
+        if isinstance(root, list):  # debug mode: evaluate labeling on explicit path samples            
+            self.cached_pipe = self._files_list_pipeline(labeler, files=root, cache=False)
+            return
+
         is_win = os.name == 'nt'
         stages = [
             CachedPipe.Source(self.config.search.scanner(self.root, is_win), 'wlk',
@@ -162,6 +175,25 @@ class DataCaster:
             serial=PickleRelativePath(self.root, safe=False), progress=progress
         )
 
+    def _files_list_pipeline(self, labeler: Labeler, files: list[str], cache: bool=False):
+        """Create a pipeline for labeling a list of files"""
+        search = self.config.search
+        method = 'search' if search.method == 'end' else 'fullmatch'
+
+        def path_labels():
+            for p in files:
+                if (m := search.parse(p, method=method)) is not None:
+                    yield m | {'path': p}
+
+        stages = [
+            CachedPipe.Source(path_labels(), 'files'),
+            CachedPipe.Map(labeler.processor, 'lbl', cfg=labeler.to_yaml()),
+        ]
+        if self.config.filters:
+            stages.append(CachedPipe.Filter(Filter(self.config.filters), 'flt',
+                                            cfg=yaml.dump(self.config.filters)))
+        return CachedPipe(stages=stages, folder=Path('.'), mode=CacheMode(cache))
+
     @property
     def labeling_namespace(self):
         from iad.core.strings import hash_str
@@ -181,6 +213,8 @@ class DataCaster:
 
     @property
     def root(self):
+        if not self.config.root:
+            raise RuntimeError(f"Root path is not set for caster '{self.config.name}'")
         return self.config.root
 
     def __repr__(self):

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/src/iad/dataman/datacast/scan.py

@@ -58,6 +58,11 @@ class GuideScan(YamlModel, hash_exclude=['_pather', '_match_skip_folders']):
     _pather: PrivateAttr = PrivateAttr()
     _match_skip_folders: PrivateAttr = PrivateAttr()
 
+    @property
+    def parse(self):
+        """Parse a string into labels using the pattern (shortcut to internal parse function)"""
+        return self._pather.regex.parse
+
     def verify_samples(self, fail=False):
         """Return list of sample failed parsing"""
         total = self.samples and len(self.samples) or 0

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/src/iad/dataman/factories.py

@@ -20,12 +20,17 @@ def create_caster(name_or_dataset=None, *, source=None, scheme=None,
 
         create_caster('ETH3D')
         create_caster(source='/data/ETH3D', scheme='ETH3D')
+        create_caster(scheme='/path/to/scheme.scm.yml', source='/data/root')
         create_caster(DatasetRM(source='/', scheme='*'))
 
     :returns: a fully-constructed DataCaster
     """
-    from .models import DatasetRM
+    from .models import DatasetRM, SchemeRM
     from .datacast.caster import DataCaster
+    from pathlib import Path
+
+    if isinstance(scheme, (str, Path)):
+        scheme = SchemeRM.from_config(scheme)
 
     config_args = dict(
         name=name_or_dataset, source=source, scheme=scheme,

ialdev_dataman-0.2.5/tests/datacast/data/res/verosnap.scm.yml

@@ -0,0 +1,26 @@
+name: verosnap
+description: test scheme
+bundle: [vru, session, capture_time]
+search:
+  pattern: >-
+    vru{vru:\d+}/images/{session:[\w-]+}/{date:[0-9_]+}/(?:{view:cam\d+}|(?:obstacle_map/occupancy_(?P<alg>grid_processed|grid))|)_?[a-z0-9-]+(?:\.(?P<kind>meta))?\.(?P<ext>jpg|json)
+
+  method: relative
+  max_depth: 7
+  samples:
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/obstacle_map/occupancy_grid_processed_e761f6da-2480-445d-8a5f-05408c4f3613.jpg
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/e761f6da-2480-445d-8a5f-05408c4f3613.meta.json
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/cam1_e761f6da-2480-445d-8a5f-05408c4f3613.jpg
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/obstacle_map/occupancy_grid_e761f6da-2480-445d-8a5f-05408c4f3613.jpg
+
+mappings:
+  alg: { 'grid': detect, grid_processed: match}
+
+if alg == 'detect' or alg == 'match':
+  view: bev
+
+if view and 'cam' in view:
+  alg: cam
+
+if ext == 'jpg':
+  kind: image

{ialdev_dataman-0.2.3 → ialdev_dataman-0.2.5}/tests/datacast/test_scheme.py

@@ -2,8 +2,11 @@ import os
 from pathlib import Path
 
 import pytest
+import yaml
 
-from iad.dataman.datacast.caster import PickleRelativePath
+from iad.dataman.datacast.caster import DataCaster, PickleRelativePath
+from iad.dataman.factories import create_caster
+from iad.dataman.models import SchemeRM
 from iad.core.tbox import TBox
 from iad.core.cache import Pickle, CacheInvalidError, CachedPipe, CacheMode
 from iad.core.fs.paths import TransPath
@@ -155,6 +158,55 @@ def test_relative_pickle_serializer(pths_gen):
     assert prp.add_root(pths_gen.file_path) == pths_gen.correct_path.__str__()
 
 
+VEROSNAP_SCHEME_YAML = Path(__file__).parent / 'data/res/verosnap.scm.yml'
+
+
+@pytest.fixture
+def verosnap_scheme():
+    return SchemeRM.from_config(VEROSNAP_SCHEME_YAML)
+
+
+def test_create_caster_from_scheme_yaml_path(tmp_path):
+    """create_caster loads SchemeRM from a .scm.yml path and builds a DataCaster."""
+    caster = create_caster(scheme=VEROSNAP_SCHEME_YAML, source=tmp_path, cache=False)
+    assert caster.config.name == 'verosnap'
+    assert len(caster.config.search.samples) == 4
+    assert caster.config.mappings['alg']['grid'] == 'detect'
+    assert caster.root == tmp_path
+
+    caster_from_str = create_caster(scheme=str(VEROSNAP_SCHEME_YAML), source=tmp_path, cache=False)
+    assert caster_from_str.config.name == caster.config.name
+    assert caster_from_str.config.search.pattern == caster.config.search.pattern
+
+
+def test_debug_mode_verosnap_samples(verosnap_scheme):
+    """DataCaster debug mode labels scheme samples without a dataset root."""
+    scheme = verosnap_scheme
+    samples = scheme.search.samples
+    caster = DataCaster(
+        name=scheme.name,
+        root=samples,
+        search=scheme.search,
+        labels=scheme.labels,
+        mappings=scheme.mappings,
+        bundle=scheme.bundle,
+    )
+    results = {lbs['path']: lbs for lbs in caster.iter()}
+    assert set(results) == set(samples)
+    expected = {
+        samples[0]: dict(vru='16', session=samples[0].split('/')[2], alg='match', view='bev', kind='image', ext='jpg'),
+        samples[1]: dict(vru='16', session=samples[1].split('/')[2], kind='meta', ext='json'),
+        samples[2]: dict(vru='16', session=samples[2].split('/')[2], alg='cam', view='cam1', kind='image', ext='jpg'),
+        samples[3]: dict(vru='16', session=samples[3].split('/')[2], alg='detect', view='bev', kind='image', ext='jpg'),
+    }
+    for path, exp in expected.items():
+        lbs = results[path]
+        for key, value in exp.items():
+            assert lbs[key] == value, f"{path}: {key}={lbs.get(key)!r}, expected {value!r}"
+        assert lbs['path'] == path
+        assert 'date' in lbs
+
+
 def test_piped_relative_pickle_serializer_(pths_gen):
     """
     Tests relative pickle behaviour when utilizes threw the caching mechanism.