ialdev-dataman 0.1.0__tar.gz → 0.2.1__tar.gz

ialdev_dataman-0.2.1/PKG-INFO

@@ -0,0 +1,82 @@
+Metadata-Version: 2.4
+Name: ialdev-dataman
+Version: 0.2.1
+Summary: Imported as `iad.dataman` — data casting, resource models, and dataset bridge
+Author: ipcoder
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: ialdev-core
+Requires-Dist: ialdev-io
+Requires-Dist: ialdev-img
+Requires-Dist: ialdev-maths
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=5.4.0
+Requires-Dist: regex>=2021.0.0
+Requires-Dist: tqdm>=4.60.0
+Requires-Dist: pytest>=7.0.0 ; extra == "dev"
+Requires-Dist: pytest-cov>=3.0.0 ; extra == "dev"
+Project-URL: Homepage, https://github.com/ipcoder/ialdev/tree/master/dataman
+Project-URL: Issues, https://github.com/ipcoder/ialdev/issues
+Project-URL: Repository, https://github.com/ipcoder/ialdev
+Provides-Extra: dev
+
+# ialdev-dataman
+
+Dataset management and data casting tools for the `iad` toolbox, published as `ialdev-dataman` and imported as `iad.dataman`.
+
+Use this package to describe datasets with resource models, discover YAML-backed dataset definitions, build data collections, apply read/transformation pipelines, and create sink repositories for generated outputs.
+
+## Install
+
+```bash
+pip install ialdev-dataman
+```
+
+Requires Python `>=3.10`, NumPy `>=2.0`, and pandas `>=2.0.0`.
+
+## Highlights
+
+- Dataset resource models: `DataSourceRM`, `SchemeRM`, `DatasetRM`, and `CollectionRM`.
+- Factory helpers: `create_caster`, `create_collection`, and `create_sink`.
+- Data collection APIs: `DataCollection`, `SinkRepo`, `CollectTable`, and `CollectSeries`.
+- YAML/Pydantic helpers through `iad.dataman.pydantools`.
+- Transform utilities for reading files into structured tables and applying image/data transforms.
+
+## Examples
+
+```python
+from iad.dataman import create_caster, create_collection
+
+caster = create_caster(
+    source="/data/images",
+    scheme="{scene}/{frame}.png",
+    filters={"scene": "scene_001"},
+)
+collection = create_collection(datasets=[caster])
+```
+
+```python
+from iad.dataman.models import DataSourceRM, DatasetRM, SchemeRM
+from iad.dataman.resman import ModelsManager
+
+manager = ModelsManager()
+manager.register_models(DataSourceRM, SchemeRM, DatasetRM)
+```
+
+```python
+from iad.dataman.pydantools import YamlModel
+
+class DatasetConfig(YamlModel):
+    name: str
+    root: str
+```
+

ialdev_dataman-0.2.1/README.md

@@ -0,0 +1,50 @@
+# ialdev-dataman
+
+Dataset management and data casting tools for the `iad` toolbox, published as `ialdev-dataman` and imported as `iad.dataman`.
+
+Use this package to describe datasets with resource models, discover YAML-backed dataset definitions, build data collections, apply read/transformation pipelines, and create sink repositories for generated outputs.
+
+## Install
+
+```bash
+pip install ialdev-dataman
+```
+
+Requires Python `>=3.10`, NumPy `>=2.0`, and pandas `>=2.0.0`.
+
+## Highlights
+
+- Dataset resource models: `DataSourceRM`, `SchemeRM`, `DatasetRM`, and `CollectionRM`.
+- Factory helpers: `create_caster`, `create_collection`, and `create_sink`.
+- Data collection APIs: `DataCollection`, `SinkRepo`, `CollectTable`, and `CollectSeries`.
+- YAML/Pydantic helpers through `iad.dataman.pydantools`.
+- Transform utilities for reading files into structured tables and applying image/data transforms.
+
+## Examples
+
+```python
+from iad.dataman import create_caster, create_collection
+
+caster = create_caster(
+    source="/data/images",
+    scheme="{scene}/{frame}.png",
+    filters={"scene": "scene_001"},
+)
+collection = create_collection(datasets=[caster])
+```
+
+```python
+from iad.dataman.models import DataSourceRM, DatasetRM, SchemeRM
+from iad.dataman.resman import ModelsManager
+
+manager = ModelsManager()
+manager.register_models(DataSourceRM, SchemeRM, DatasetRM)
+```
+
+```python
+from iad.dataman.pydantools import YamlModel
+
+class DatasetConfig(YamlModel):
+    name: str
+    root: str
+```

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/docs/use_cases.md

@@ -12,7 +12,7 @@ Resource types implemented as subclasses of pydantic model ``ResourseModel``
 
 ```python
 from iad.dataman.resman import ResourceModel, locatable
-from iad.core.env import EnvLoc
+from iad.core.fs.env import EnvLoc
 
 
 @locatable(EnvLoc.RESOURCES / 'houses')

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/pyproject.toml

@@ -4,11 +4,14 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "ialdev-dataman"
-version = "0.1.0"
-description = "iad.dataman — data casting, resource models, and dataset bridge"
+version = "0.2.1"
+description = "Imported as `iad.dataman` — data casting, resource models, and dataset bridge"
 readme = "README.md"
 requires-python = ">=3.10"
 license = {text = "MIT"}
+authors = [
+    {name = "ipcoder"}
+]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
@@ -24,8 +27,8 @@ dependencies = [
     "ialdev-io",
     "ialdev-img",
     "ialdev-maths",
-    "pandas>=1.3.0",
-    "numpy>=1.20.0",
+    "pandas>=2.0.0",
+    "numpy>=2.0",
     "pydantic>=2.0",
     "pyyaml>=5.4.0",
     "regex>=2021.0.0",
@@ -38,6 +41,11 @@ dev = [
     "pytest-cov>=3.0.0",
 ]
 
+[project.urls]
+Homepage = "https://github.com/ipcoder/ialdev/tree/master/dataman"
+Repository = "https://github.com/ipcoder/ialdev"
+Issues = "https://github.com/ipcoder/ialdev/issues"
+
 [tool.flit.module]
 name = "iad.dataman"
 

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/__init__.py

@@ -4,7 +4,7 @@ from .models import DataSourceRM, SchemeRM, DatasetRM, CollectionRM, DSample, di
 from .factories import create_caster, create_collection, create_sink
 from .datacast.collect import DataCollection, SinkRepo
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 __all__ = [
     "__version__",

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/datacast/caster.py

@@ -8,12 +8,12 @@ from pydantic.v1 import PrivateAttr
 import regex as re  # Using more advanced regex package!
 import yaml
 
-from iad.core.param import YamlModel
+from iad.dataman.pydantools import YamlModel
 from iad.core import logger, as_list
 from iad.core.cache import CachedPipe, CacheMode, Pickle, CacheInvalidError, NoSerial
 from iad.core.datatools import Filter
 from iad.core.events import Timer
-from iad.core.filesproc import Path, root_cropper, root_adder, normalize
+from iad.core.fs.filesproc import Path, root_cropper, root_adder, normalize
 from iad.core.fnctools import express_to_kw_func
 from .scan import GuideScan
 
@@ -238,7 +238,7 @@ class DataCaster:
             _log.warning(f"{self} found no matching files!")
 
         if cfg := self.config.sample:
-            from iad.core.pdtools import sample
+            from iad.core.data.pdtools import sample
             ds = sample(ds, **(cfg if isinstance(cfg, dict) else cfg.dict()))
         return ds
 

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/datacast/collect.py

@@ -10,7 +10,7 @@ with Timer(f" ← {__file__} imports", "timing", min=0.1, pre=f' → importing i
 
     # utils import
     from iad.core import as_list, logger, drop_undef, as_iter
-    import iad.core.pdtools as pdt
+    import iad.core.data.pdtools as pdt
     from iad.core.cache import CacheMode, Cacher
 
     # internal imports

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/datacast/labeled.py

@@ -48,7 +48,7 @@ class LabelRules:
     @staticmethod
     def load_domains(path=None):
         """Load domains conventions"""
-        from iad.core.param import TBox
+        from iad.core.tbox import TBox
         from iad.dataman.env import EnvLoc
         # Consider: is that the right approach ? What if RES_LOC missing and path is None?
         path = path or (EnvLoc.RESOURCES / 'schemes').first_file('labels.yml')

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/datacast/scan.py

@@ -8,10 +8,10 @@ import pydantic.v1 as pydantic
 import regex as re
 from pydantic.v1 import PrivateAttr
 
-from iad.core.param import YamlModel
+from iad.dataman.pydantools import YamlModel
 from iad.core import logger
-from iad.core.filesproc import normalize
-from iad.core.paths import TransPath
+from iad.core.fs.filesproc import normalize
+from iad.core.fs.paths import TransPath
 
 _log = logger('datacast.scan')
 

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/datacast/transtools.py

@@ -7,9 +7,10 @@ from typing import Union, Callable, Iterable, Literal, Any, Collection, get_args
 import numpy as np
 import pandas as pd
 
-from iad.core import as_list, pdtools as pdt
+from iad.core import as_list
+import iad.core.data.pdtools as pdt
 from iad.core.fnctools import O, Namespace
-from iad.core.label import Labels
+from iad.core.data.label import Labels
 from iad.core.wrap import name_tuple
 
 EMPTY_CELL = (None, pd.NA, np.nan)

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/env.py

@@ -1,4 +1,4 @@
-from iad.core.env import EnvLoc as CoreEnvLoc, Locator, env_log
+from iad.core.fs.env import EnvLoc as CoreEnvLoc, Locator, env_log
 
 
 class EnvLoc(CoreEnvLoc):

{ialdev_dataman-0.1.0 → ialdev_dataman-0.2.1}/src/iad/dataman/models.py

@@ -230,7 +230,7 @@ class Selection:
         if isinstance(v, str):
             if re.fullmatch(r'\d+(\.\d*)?%', v):
                 v = float(v[:-1]) / 100
-            elif re.fullmatch('\([\w, +-]+\)', v):
+            elif re.fullmatch(r'\([\w, +-]+\)', v):
                 v = eval('slice' + v)
             else:
                 ValueError(f'Invalid sample selection value: "{v}"')

ialdev_dataman-0.2.1/src/iad/dataman/pydantools/__init__.py

@@ -0,0 +1,5 @@
+"""Pydantic + YAML model utilities for ``iad.dataman``."""
+
+from iad.dataman.pydantools.models import YamlModel, make_yml_model_format, model_arguments
+
+__all__ = ["YamlModel", "make_yml_model_format", "model_arguments"]

ialdev_dataman-0.2.1/src/iad/dataman/pydantools/fixed_pydantic_yaml/__init__.py

@@ -0,0 +1,32 @@
+"""YAML-enabled Pydantic models."""
+
+from .version import __version__
+
+
+__all__ = [
+    "__version__",
+    "SemVer",
+    "yaml",
+    "VersionedYamlModel",
+    "YamlEnum",  # deprecated class
+    "YamlInt",
+    "YamlIntEnum",
+    "YamlModel",
+    "YamlModelMixin",
+    "YamlModelMixinConfig",
+    "YamlStr",
+    "YamlStrEnum",
+]
+from .main import (
+    SemVer,
+    VersionedYamlModel,
+    YamlEnum,
+    YamlInt,
+    YamlIntEnum,
+    YamlModel,
+    YamlModelMixin,
+    YamlModelMixinConfig,
+    YamlStr,
+    YamlStrEnum,
+    yaml,
+)

ialdev_dataman-0.2.1/src/iad/dataman/pydantools/fixed_pydantic_yaml/compat/hacks.py

@@ -0,0 +1,76 @@
+"""Module for some hacks that are needed to work around the YAML library interfaces.
+
+Mainy, this defines "safe" representers for common types that Pydantic is able to parse.
+"""
+
+__all__ = ["inject_all"]
+
+from typing import List, Type
+
+from .representers import register_str_like, register_int_like
+
+
+def get_str_like_types() -> List[Type]:
+    """Returns many string-like types from stdlib and Pydantic."""
+
+    # flake8: noqa
+
+    from uuid import UUID
+    from pathlib import (
+        Path,
+        PosixPath,
+        PurePath,
+        PurePosixPath,
+        PureWindowsPath,
+        WindowsPath,
+    )
+
+    from pydantic.v1 import types, networks
+
+    def _chk(x) -> bool:
+        try:
+            return issubclass(x, (str, Path, PurePath, UUID))
+        except Exception:
+            return False
+
+    # Get most candidates
+    candidates = (
+        list(locals().values())
+        + [getattr(types, v) for v in types.__all__]
+        + [getattr(networks, v) for v in networks.__all__]
+    )
+    str_like = [v for v in candidates if _chk(v)]
+
+    # SecretStr, SecretBytes are stringified as "**********" by Pydantic
+    str_like += [types.SecretStr, types.SecretBytes]
+    return str_like
+
+
+def get_int_like_types() -> List[Type]:
+    """Returns many int-like types from stdlib and Pydantic."""
+
+    from pydantic.v1 import types
+
+    def _chk(x) -> bool:
+        try:
+            return issubclass(x, (int))
+        except Exception:
+            return False
+
+    candidates = list(locals().values()) + [getattr(types, v) for v in types.__all__]
+    int_like = [v for v in candidates if _chk(v)]
+    return int_like
+
+
+def inject_all():
+    """Injects all necessary "hacks" into the namespace.
+
+    What this currently does:
+    - Registers many str-like and int-like types (from Pydantic and the standard
+      library) to be representable in YAML.
+    """
+    # Add representers for string-like and int-like values.
+    for cls in get_str_like_types():
+        register_str_like(cls, method=str)
+    for cls in get_int_like_types():
+        register_int_like(cls, method=int)

ialdev_dataman-0.2.1/src/iad/dataman/pydantools/fixed_pydantic_yaml/compat/old_enums.py

@@ -0,0 +1,37 @@
+"""Old class for YamlEnum that is now deprecated (and marked as such)."""
+
+from enum import Enum
+from inspect import isabstract
+
+from deprecated import deprecated
+
+from .representers import register_str_like
+
+__all__ = ["YamlEnum"]
+
+
+@deprecated(version="0.5.0", reason="Use the `YamlStrEnum` class instead.")
+class YamlEnum(Enum):
+    """This class is DEPRECATED, please use `pydantic_yaml.YamlStrEnum` instead.
+
+    Enumeration that serializes as the proper underlying object type.
+
+    You can use this instead of `enum.Enum`, for example:
+        class MyEnum(str, YamlEnum):
+            val1 = "Value 1"
+            val2 = "Value 2"
+
+    Note
+    ----
+    This is actually a lie, it only supports `str` enums for now. Oops.
+    """
+
+    def __init_subclass__(cls):
+        if not isabstract(cls):
+            register_str_like(cls)
+
+    def __repr__(self) -> str:
+        return repr(self.value)
+
+    def __str__(self) -> str:
+        return str(self.value)

ialdev_dataman-0.2.1/src/iad/dataman/pydantools/fixed_pydantic_yaml/compat/representers.py

@@ -0,0 +1,92 @@
+"""Representers for dumping common objects to YAML."""
+
+from functools import partial
+from typing import Any, Callable, TypeVar
+
+from .yaml_lib import yaml, dumper_classes, representer_classes
+
+CType = TypeVar("CType")
+
+__all__ = ["register_str_like", "register_int_like"]
+
+
+def dump_as_str(dumper: yaml.Dumper, data: Any, method: Callable[[Any], str] = str):
+    """Represents an object as a string in YAML.
+
+    Parameters
+    ----------
+    dumper : yaml.Dumper
+        The dumper instance, such as `yaml.SafeDumper()`.
+    data
+        The object to dump. Ideally this is string-like.
+    method : callable
+        A method that converts `data` to a string. Default is `str`.
+
+    Returns
+    -------
+    node
+        The YAML internal node representation.
+    """
+    return dumper.represent_str(method(data))
+
+
+def dump_as_int(dumper: yaml.Dumper, data: Any, method: Callable[[Any], int] = int):
+    """Represents an object as an integer in YAML.
+
+    Parameters
+    ----------
+    dumper : yaml.Dumper
+        The dumper instance, such as `yaml.SafeDumper()`.
+    data
+        The object to dump. Ideally this is int-like.
+    method : callable
+        A method that converts `data` to an integer. Default is `int`.
+
+    Returns
+    -------
+    node
+        The YAML internal node representation.
+    """
+    return dumper.represent_int(method(data))
+
+
+def register_str_like(cls: CType, method: Callable[[Any], str] = str) -> CType:
+    """Registers `cls` to be dumped to YAML as a string.
+
+    Parameters
+    ----------
+    cls : Type
+        The class to represent.
+    method : callable
+        A method that converts objects of type `cls` to a string.
+        Default is `str`.
+
+    Returns
+    -------
+    cls
+        This is the same as the input `cls`.
+    """
+    for x_cls in dumper_classes + representer_classes:
+        x_cls.add_representer(cls, partial(dump_as_str, method=method))
+    return cls
+
+
+def register_int_like(cls: CType, method: Callable[[Any], int] = int) -> CType:
+    """Registers `cls` to be dumped to YAML as an integer.
+
+    Parameters
+    ----------
+    cls : Type
+        The class to represent.
+    method : callable
+        A method that converts objects of type `cls` to an integer.
+        Default is `int`.
+
+    Returns
+    -------
+    cls
+        This is the same as the input `cls`.
+    """
+    for x_cls in dumper_classes + representer_classes:
+        x_cls.add_representer(cls, partial(dump_as_int, method=method))
+    return cls

ialdev_dataman-0.2.1/src/iad/dataman/pydantools/fixed_pydantic_yaml/compat/types.py

@@ -0,0 +1,122 @@
+"""Defines useful types for building your own YAML-compatible types."""
+
+from enum import Enum
+from inspect import isabstract
+from typing import Dict, Tuple, Union
+
+from .representers import register_int_like, register_str_like
+from .yaml_lib import yaml_safe_dump
+
+__all__ = ["YamlInt", "YamlIntEnum", "YamlStr", "YamlStrEnum"]
+
+
+class YamlStr(str):
+    """A `str` subclass that serializes to YAML as an integer."""
+
+    def __init_subclass__(cls):
+        """Adds a representer when defining a subclass."""
+        res = super().__init_subclass__()
+        if not isabstract(cls):
+            register_str_like(cls, method=cls._to_yaml_str)
+        return res
+
+    @classmethod
+    def _to_yaml_str(cls, v) -> str:
+        """Represent a value of this type as a string."""
+        return str(v)
+
+    def __str__(self) -> str:
+        """This just returns the wrapped string."""
+        return super().__str__()
+
+    def __repr__(self) -> str:
+        """Repr string that keeps the class type. You may redefine this."""
+        return type(self).__qualname__ + "(" + super().__repr__() + ")"
+
+    def __getnewargs__(self) -> Tuple[str]:
+        """Implementation to prevent Enum from sabotaging us.
+
+        I'm not joking! See `_make_class_unpicklable()` from `enum.py`
+        """
+        return super().__getnewargs__()
+
+
+class YamlInt(int):
+    """An `int` subclass that serializes to YAML as an integer."""
+
+    def __init_subclass__(cls):
+        """Adds a representer when defining a subclass."""
+        res = super().__init_subclass__()
+        if not isabstract(cls):
+            register_int_like(cls, method=cls._to_yaml_int)
+        return res
+
+    @classmethod
+    def _to_yaml_int(cls, v) -> int:
+        """Represent a value of this type as an integer."""
+        return int(v)
+
+    def __int__(self) -> int:
+        return super().__int__()
+
+    def __str__(self) -> str:
+        """This returns the wrapped int."""
+        return super().__str__()
+
+    def __repr__(self) -> str:
+        """Repr string that keeps the class type. You may redefine this."""
+        return type(self).__qualname__ + "(" + super().__repr__() + ")"
+
+    def __getnewargs__(self) -> Tuple[int]:
+        """Implementation to prevent Enum from sabotaging us.
+
+        I'm not joking! See `_make_class_unpicklable()` from `enum.py`
+        """
+        return super().__getnewargs__()
+
+
+class YamlStrEnum(YamlStr, Enum):
+    """String-valued enum that serializes to YAML directly as a string.
+
+    This also checks that your enum can properly be serialized.
+    """
+
+    def __init_subclass__(cls):
+        res = super().__init_subclass__()
+        if not isabstract(cls):
+            vals: Dict[str, cls] = dict(cls.__members__)
+            yaml_safe_dump(vals)
+        return res
+
+    def __str__(self) -> str:
+        """This returns the wrapped string."""
+        return str.__str__(self)
+
+    def __repr__(self) -> str:
+        """Repr string that keeps the class type. You may redefine this."""
+        return type(self).__qualname__ + "(" + str.__repr__(self) + ")"
+
+
+class YamlIntEnum(YamlInt, Enum):
+    """Integer-valued enum that serializes to YAML directly as an integer.
+
+    This also checks that your enum can properly be serialized.
+    """
+
+    def __init_subclass__(cls):
+        res = super().__init_subclass__()
+        if not isabstract(cls):
+            vals: Dict[str, cls] = dict(cls.__members__)
+            yaml_safe_dump(vals)
+        return res
+
+
+if __name__ == "__main__":
+
+    class A(YamlStrEnum):
+        a = "a"
+        b = "b"
+
+    class X(YamlIntEnum):
+        x1 = 1
+        x2 = 2