ialdev-dataman 0.2.4__tar.gz → 0.2.6__tar.gz

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ialdev-dataman
-Version: 0.2.4
+Version: 0.2.6
 Summary: Imported as `iad.dataman` — data casting, resource models, and dataset bridge
 Author: ipcoder
 Requires-Python: >=3.10
@@ -12,10 +12,10 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: ialdev-core>=0.2.4
-Requires-Dist: ialdev-io>=0.2.2
-Requires-Dist: ialdev-img>=0.2.2
-Requires-Dist: ialdev-maths>=0.2.3
+Requires-Dist: ialdev-core>=0.2.5
+Requires-Dist: ialdev-io>=0.2.3
+Requires-Dist: ialdev-img>=0.2.3
+Requires-Dist: ialdev-maths>=0.2.4
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: numpy>=1.26.4,<2
 Requires-Dist: pydantic>=2.0

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "ialdev-dataman"
-version = "0.2.4"
+version = "0.2.6"
 description = "Imported as `iad.dataman` — data casting, resource models, and dataset bridge"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -23,10 +23,10 @@ classifiers = [
 ]
 
 dependencies = [
-    "ialdev-core>=0.2.4",
-    "ialdev-io>=0.2.2",
-    "ialdev-img>=0.2.2",
-    "ialdev-maths>=0.2.3",
+    "ialdev-core>=0.2.5",
+    "ialdev-io>=0.2.3",
+    "ialdev-img>=0.2.3",
+    "ialdev-maths>=0.2.4",
     "pandas>=2.0.0",
     "numpy>=1.26.4,<2",
     "pydantic>=2.0",
@@ -51,6 +51,7 @@ name = "iad.dataman"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "--import-mode=importlib"
 python_files = ["test_*.py"]
 python_classes = ["Test*"]
 python_functions = ["test_*"]

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/src/iad/dataman/__init__.py

@@ -4,7 +4,7 @@ from .models import DataSourceRM, SchemeRM, DatasetRM, CollectionRM, DSample, di
 from .factories import create_caster, create_collection, create_sink
 from .datacast.collect import DataCollection, SinkRepo
 
-__version__ = "0.2.4"
+__version__ = "0.2.6"
 
 __all__ = [
     "__version__",

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/src/iad/dataman/datacast/caster.py

@@ -32,13 +32,13 @@ class CasterConfig:
     __slots__ = ('name', 'root', 'search', 'labels', 'mappings',
                  'reverse', 'bundle', 'filters', 'sample')
 
-    def __init__(self, *, name: str = None, root: Path | str,
+    def __init__(self, *, name: str = None, root: Path | str | None,
                  search: GuideScan, labels: dict = None,
                  mappings: dict = None, reverse: bool = False,
                  bundle: list | None = None, filters: dict = None,
                  sample: dict = None):
         self.name = name
-        self.root = Path(root) if root else None
+        self.root = root and Path(root)
         self.search = search
         self.labels = labels or {}
         self.mappings = mappings or {}
@@ -116,7 +116,7 @@ class DataCaster:
     CacheMode = CacheMode  # allow PathScheme.CacheMode instead of scheme.CacheMode
 
     def __init__(self, name: str = None, *,
-                 root: Path | str,
+                 root: Path | str | list[str],
                  search: GuideScan,
                  labels: dict = None,
                  mappings: dict = None,
@@ -128,16 +128,24 @@ class DataCaster:
                  temp_cache=False, cache: Optional[CacheMode | bool] = True,
                  progress: dict | bool = None):
         """
-        Create data caster semantic parser for particularly structured files tree.
+        Create a data caster semantic parser for a particularly structured files tree.
         ::
             tree -> [pattern] -> files -> [label] -> [filter] -> labeled
 
         All parameters must be fully resolved (no name-based lookup).
-        For name-based construction (e.g. ``DataCaster('ETH3D')``), use the
-        factory in ``dataman`` (e.g. ``dataman.create_caster``).
+        For name-based construction (e.g. `DataCaster('ETH3D')`), use the
+        factory in `dataman` (e.g. ``dataman.create_caster``).
+
+        **Debug Mode**
+        There is a special "debug" mode, when instead of a root of the tree, a list of paths is provided.
+        That allows evaluating the labeling pipeline on a specific set of files
+        without the need for a full tree structure.
+        (`GuideScan.samples` may be used as such input if the scheme defines them)
+        In this case caching is always disabled.
         """
         self.config = CasterConfig(
-            name=name, root=root, search=search, labels=labels,
+            name=name, root=root if isinstance(root, (Path, str)) else '',
+            search=search, labels=labels,
             mappings=mappings, reverse=reverse, bundle=bundle,
             filters=filters, sample=sample,
         )
@@ -146,6 +154,11 @@ class DataCaster:
                           reverse=self.config.reverse, namespace=self.labeling_namespace)
         self.categories = labeler.categories
         self.bundle = self.config.bundle or []
+
+        if isinstance(root, list):  # debug mode: evaluate labeling on explicit path samples            
+            self.cached_pipe = self._files_list_pipeline(labeler, files=root, cache=False)
+            return
+
         is_win = os.name == 'nt'
         stages = [
             CachedPipe.Source(self.config.search.scanner(self.root, is_win), 'wlk',
@@ -162,6 +175,25 @@ class DataCaster:
             serial=PickleRelativePath(self.root, safe=False), progress=progress
         )
 
+    def _files_list_pipeline(self, labeler: Labeler, files: list[str], cache: bool=False):
+        """Create a pipeline for labeling a list of files"""
+        search = self.config.search
+        method = 'search' if search.method == 'end' else 'fullmatch'
+
+        def path_labels():
+            for p in files:
+                if (m := search.parse(p, method=method)) is not None:
+                    yield m | {'path': p}
+
+        stages = [
+            CachedPipe.Source(path_labels(), 'files'),
+            CachedPipe.Map(labeler.processor, 'lbl', cfg=labeler.to_yaml()),
+        ]
+        if self.config.filters:
+            stages.append(CachedPipe.Filter(Filter(self.config.filters), 'flt',
+                                            cfg=yaml.dump(self.config.filters)))
+        return CachedPipe(stages=stages, folder=Path('.'), mode=CacheMode(cache))
+
     @property
     def labeling_namespace(self):
         from iad.core.strings import hash_str
@@ -181,6 +213,8 @@ class DataCaster:
 
     @property
     def root(self):
+        if not self.config.root:
+            raise RuntimeError(f"Root path is not set for caster '{self.config.name}'")
         return self.config.root
 
     def __repr__(self):

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/src/iad/dataman/datacast/scan.py

@@ -58,6 +58,11 @@ class GuideScan(YamlModel, hash_exclude=['_pather', '_match_skip_folders']):
     _pather: PrivateAttr = PrivateAttr()
     _match_skip_folders: PrivateAttr = PrivateAttr()
 
+    @property
+    def parse(self):
+        """Parse a string into labels using the pattern (shortcut to internal parse function)"""
+        return self._pather.regex.parse
+
     def verify_samples(self, fail=False):
         """Return list of sample failed parsing"""
         total = self.samples and len(self.samples) or 0

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/src/iad/dataman/datacast/transtools.py

@@ -222,16 +222,20 @@ def apply_column_transform(df: pd.Series | pd.DataFrame, *,
             except:
                 return fallback(g)
 
-    df_out[out] = df_out.apply(func, axis=1)
+    # These are deliberate writes into the working frame; suppress the
+    # false-positive SettingWithCopyWarning when the caller passed a frame
+    # that pandas flags as a copy-of-slice (e.g. a `.qix()`/`.select()` result).
+    with pd.option_context('mode.chained_assignment', None):
+        df_out.loc[:, out] = df_out.apply(func, axis=1)
 
-    if drop is True:
-        drop = set(inputs) - {out} | {trans}
-    if drop:
-        if not inplace:
-            df.drop(drop, axis=1, inplace=True)  # drop used columns in the original df, if returning new df
-            return df_out.drop(drop, axis=1)
-        df_out.drop(drop, axis=1, inplace=True)  # drop used columns in the inplace df
-    return df_out
+        if drop is True:
+            drop = set(inputs) - {out} | {trans}
+        if drop:
+            if not inplace:
+                df.drop(drop, axis=1, inplace=True)  # drop used columns in the original df, if returning new df
+                return df_out.drop(drop, axis=1)
+            df_out.drop(drop, axis=1, inplace=True)  # drop used columns in the inplace df
+        return df_out
 
 
 def row_func_prepend_transform(fnc: Callable, trans=Col.read_trans, inp=Col.path, out=Col.data):
@@ -533,7 +537,7 @@ class Fetchable:
         trans_labels = [Col.path, Col.read_trans]
 
         if self.empty:
-            self['data'] = None
+            self.loc[:, 'data'] = None
             return self
 
         if not isinstance(self, pd.DataFrame):  # Series, must be already with data
@@ -551,7 +555,10 @@ class Fetchable:
         if reuse or check_file:  # change to conditional read if required
             kws = {'check_file': check_file} | (reuse and {'check_data': reuse} or {})
             trans = '__cond_trans__'
-            self[trans] = O(cond_read_trans, **kws)
+            # Deliberate write into self (fetch documents in-place data loading);
+            # suppress the false-positive copy-of-slice warning.
+            with pd.option_context('mode.chained_assignment', None):
+                self.loc[:, trans] = O(cond_read_trans, **kws)
             trans_args = dict(drop=trans, args='series', inp=None)  # series with all columns
         else:
             trans = Col.read_trans

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/src/iad/dataman/factories.py

@@ -20,12 +20,17 @@ def create_caster(name_or_dataset=None, *, source=None, scheme=None,
 
         create_caster('ETH3D')
         create_caster(source='/data/ETH3D', scheme='ETH3D')
+        create_caster(scheme='/path/to/scheme.scm.yml', source='/data/root')
         create_caster(DatasetRM(source='/', scheme='*'))
 
     :returns: a fully-constructed DataCaster
     """
-    from .models import DatasetRM
+    from .models import DatasetRM, SchemeRM
     from .datacast.caster import DataCaster
+    from pathlib import Path
+
+    if isinstance(scheme, (str, Path)):
+        scheme = SchemeRM.from_config(scheme)
 
     config_args = dict(
         name=name_or_dataset, source=source, scheme=scheme,

ialdev_dataman-0.2.6/tests/datacast/data/res/verosnap.scm.yml

@@ -0,0 +1,26 @@
+name: verosnap
+description: test scheme
+bundle: [vru, session, capture_time]
+search:
+  pattern: >-
+    vru{vru:\d+}/images/{session:[\w-]+}/{date:[0-9_]+}/(?:{view:cam\d+}|(?:obstacle_map/occupancy_(?P<alg>grid_processed|grid))|)_?[a-z0-9-]+(?:\.(?P<kind>meta))?\.(?P<ext>jpg|json)
+
+  method: relative
+  max_depth: 7
+  samples:
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/obstacle_map/occupancy_grid_processed_e761f6da-2480-445d-8a5f-05408c4f3613.jpg
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/e761f6da-2480-445d-8a5f-05408c4f3613.meta.json
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/cam1_e761f6da-2480-445d-8a5f-05408c4f3613.jpg
+    - vru16/images/2026-05-10_12_16_26_b63303ca57/2026_05_10_08_33_23/obstacle_map/occupancy_grid_e761f6da-2480-445d-8a5f-05408c4f3613.jpg
+
+mappings:
+  alg: { 'grid': detect, grid_processed: match}
+
+if alg == 'detect' or alg == 'match':
+  view: bev
+
+if view and 'cam' in view:
+  alg: cam
+
+if ext == 'jpg':
+  kind: image

{ialdev_dataman-0.2.4 → ialdev_dataman-0.2.6}/tests/datacast/test_scheme.py

@@ -2,8 +2,11 @@ import os
 from pathlib import Path
 
 import pytest
+import yaml
 
-from iad.dataman.datacast.caster import PickleRelativePath
+from iad.dataman.datacast.caster import DataCaster, PickleRelativePath
+from iad.dataman.factories import create_caster
+from iad.dataman.models import SchemeRM
 from iad.core.tbox import TBox
 from iad.core.cache import Pickle, CacheInvalidError, CachedPipe, CacheMode
 from iad.core.fs.paths import TransPath
@@ -155,6 +158,55 @@ def test_relative_pickle_serializer(pths_gen):
     assert prp.add_root(pths_gen.file_path) == pths_gen.correct_path.__str__()
 
 
+VEROSNAP_SCHEME_YAML = Path(__file__).parent / 'data/res/verosnap.scm.yml'
+
+
+@pytest.fixture
+def verosnap_scheme():
+    return SchemeRM.from_config(VEROSNAP_SCHEME_YAML)
+
+
+def test_create_caster_from_scheme_yaml_path(tmp_path):
+    """create_caster loads SchemeRM from a .scm.yml path and builds a DataCaster."""
+    caster = create_caster(scheme=VEROSNAP_SCHEME_YAML, source=tmp_path, cache=False)
+    assert caster.config.name == 'verosnap'
+    assert len(caster.config.search.samples) == 4
+    assert caster.config.mappings['alg']['grid'] == 'detect'
+    assert caster.root == tmp_path
+
+    caster_from_str = create_caster(scheme=str(VEROSNAP_SCHEME_YAML), source=tmp_path, cache=False)
+    assert caster_from_str.config.name == caster.config.name
+    assert caster_from_str.config.search.pattern == caster.config.search.pattern
+
+
+def test_debug_mode_verosnap_samples(verosnap_scheme):
+    """DataCaster debug mode labels scheme samples without a dataset root."""
+    scheme = verosnap_scheme
+    samples = scheme.search.samples
+    caster = DataCaster(
+        name=scheme.name,
+        root=samples,
+        search=scheme.search,
+        labels=scheme.labels,
+        mappings=scheme.mappings,
+        bundle=scheme.bundle,
+    )
+    results = {lbs['path']: lbs for lbs in caster.iter()}
+    assert set(results) == set(samples)
+    expected = {
+        samples[0]: dict(vru='16', session=samples[0].split('/')[2], alg='match', view='bev', kind='image', ext='jpg'),
+        samples[1]: dict(vru='16', session=samples[1].split('/')[2], kind='meta', ext='json'),
+        samples[2]: dict(vru='16', session=samples[2].split('/')[2], alg='cam', view='cam1', kind='image', ext='jpg'),
+        samples[3]: dict(vru='16', session=samples[3].split('/')[2], alg='detect', view='bev', kind='image', ext='jpg'),
+    }
+    for path, exp in expected.items():
+        lbs = results[path]
+        for key, value in exp.items():
+            assert lbs[key] == value, f"{path}: {key}={lbs.get(key)!r}, expected {value!r}"
+        assert lbs['path'] == path
+        assert 'date' in lbs
+
+
 def test_piped_relative_pickle_serializer_(pths_gen):
     """
     Tests relative pickle behaviour when utilizes threw the caching mechanism.