ialdev-dataman 0.1.0__tar.gz

ialdev_dataman-0.1.0/PKG-INFO

@@ -0,0 +1,33 @@
+Metadata-Version: 2.4
+Name: ialdev-dataman
+Version: 0.1.0
+Summary: iad.dataman — data casting, resource models, and dataset bridge
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: ialdev-core
+Requires-Dist: ialdev-io
+Requires-Dist: ialdev-img
+Requires-Dist: ialdev-maths
+Requires-Dist: pandas>=1.3.0
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: pyyaml>=5.4.0
+Requires-Dist: regex>=2021.0.0
+Requires-Dist: tqdm>=4.60.0
+Requires-Dist: pytest>=7.0.0 ; extra == "dev"
+Requires-Dist: pytest-cov>=3.0.0 ; extra == "dev"
+Provides-Extra: dev
+
+# ialdev-dataman
+
+`iad.dataman` — data casting, resource models, and dataset bridge.
+
+Install: `pip install ialdev-dataman`
+

ialdev_dataman-0.1.0/README.md

@@ -0,0 +1,5 @@
+# ialdev-dataman
+
+`iad.dataman` — data casting, resource models, and dataset bridge.
+
+Install: `pip install ialdev-dataman`

ialdev_dataman-0.1.0/docs/use_cases.md

@@ -0,0 +1,110 @@
+# Resource Management - Use Cases
+
+First make sure you are familiar with the main [concepts](./concepts.md).
+
+# Resources Life Timeline
+
+### Resource Type Definition
+Every resource is an instance of certain _resource type_ which must be defined first.
+
+Resource types implemented as subclasses of pydantic model ``ResourseModel``
+(thus may be called resource models).
+
+```python
+from iad.dataman.resman import ResourceModel, locatable
+from iad.core.env import EnvLoc
+
+
+@locatable(EnvLoc.RESOURCES / 'houses')
+class HouseModel(ResourceModel):
+    rooms: int
+    area: float
+    ...
+
+```
+There are two stages in introducing resources into the framework:
+
+1. Define a *locatable* Resource Model
+2. Discover resources in the defined locations
+
+### Creation
+Resources can be created:
+ 1. explicitly as `*.yml` _files_
+ 2. explicitly as _instances_ of the corresponding `pydantic` model class
+ 3. implicitly during initialization of other resource as default attributes
+
+### Discovery of Filed Resources
+To be accessibly through the `resman` API the file-form resources must
+be first _discovered_ and initialized as _instances_.
+
+Discovery of resources is responsibility resource managers of their particular type. \
+Those managers are created automatically _when a new class of resource Model is imported_.
+``ModelsManager`` class can be used as a single access point to the resource management system.
+
+For example, lets assume there is
+
+```python
+from inu.resman import ModelsManager  # initialize the resource management sub-system
+
+# At this stage no specific resource types are registered in the ModelsManager!
+
+assert not ModelsManager.list()
+# Let's import model of datasets management resources defined by the ``datacast`` package.
+from inu.datacast import models as dm
+
+# Now all the resource Models (Datasource, Scheme, Dataset, DataCollection)
+# from there are registered in the ModelManager
+assert ModelsManager.list() > 3
+
+# Manger of specific resource type can be found by its model class or name
+ds_man = ModelsManager.get(dm.DatasetRM)
+assert ds_man is ModelsManager.get('dataset')
+
+# Discovery of its resources are can be requested
+ds_man.discover()  # without arguments look in the default locations
+
+# There is a shortcut version of this sequence:
+ModelsManager.discover('dataset')
+
+# Or, to discover resources from all the currently registered models
+ModelsManager.discover()
+```
+
+
+## Scenarios
+
+### Initialization
+
+#### Access Resource Models
+
+```python
+from inu.datacast import ModelsManager, ResManager, find_resource
+
+dsm: ResManager = ModelsManager.find("dataset")
+assert dsm is None, "DatasetRM is not been imported yet"
+
+from inu.datacast import models
+# that imports among the other DatasetRM
+dsm = find_resource('dataset')
+assert isinstance(dsm, ResManager)
+```
+
+#### Scan Resource Folders
+
+```python
+from inu.datacast.models import DatasetRM
+from inu.resman import ModelsManager
+
+dsm = ModelsManager.find(DatasetRM)
+assert dsm.list().empty, "Never scanned"
+assert dsm.folders(), "Some Folders are defined with Model"
+assert any(p.is_dir() for p in dsm.folders()), "Some folders must exist"
+
+dsm.discover()  # assuming there are resources there
+assert not dsm.list().empty
+
+
+
+```
+
+

ialdev_dataman-0.1.0/docs/use_cases.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+# FixMe: Do we still need that?
+
+from typing import Optional
+
+from iad.dataman.env import EnvLoc
+from iad.dataman.resman import ModelsManager, ResourceModel  # initialize the resource management sub-system
+from iad.core.logs import setup_logs, logger
+
+
+mm = ModelsManager()
+
+
+_log = logger('use_cases')
+lof_func_enable = False
+
+
+class MyRes(ResourceModel):
+    description: str = "Hi"
+
+
+class MyRes(ResourceModel):
+    description: str = "Hi"
+
+
+def log_func(f):
+    def wrapper(*args, **kwargs):
+        _log.warning(f"------------------- {f.__name__.upper()} ---------------------")
+        return f(*args, **kwargs)
+    return wrapper if lof_func_enable else f
+
+
+def case_create_resource():
+    class ResAModel(ResourceModel):
+        description: Optional[str]
+
+    ra1 = ResAModel('a1', description='Wrong')
+    assert ra1.description
+
+
+@log_func
+def case_init_models_management():
+    import iad.dataman.models as dm
+    # At this stage no specific resource types are registered in the mm!
+    mm._reset()      # but we reset anyway for robustness of the test
+    assert not mm.list()
+    # Let's import model of datasets management resources defined by the ``datasets`` bridge.
+    mm.register_models(dm)
+    # Now all the resource Models (Datasource, Scheme, Dataset, DataCollection)
+    # from there are registered in the ModelManager
+    assert len(mm.list()) > 3
+
+    # Manger of specific resource type can be found by its model class or name
+    datasets = mm.get(dm.DatasetRM)
+    assert datasets is mm.get('dataset')  # search by name
+
+
+@log_func
+def case_resource_creation():
+
+    from iad.dataman.models import DataSourceRM
+    src = DataSourceRM.parse_file(EnvLoc.DATA / 'datasource.yml')  # config from yaml file - Standard
+
+    data_root = EnvLoc.DATA.first_existing()
+    # From config dict
+    cfg = {'name': 'SourceName', 'root': data_root}
+    src = DataSourceRM.parse_obj(cfg)  # 1. config from dict - pydantic
+    src = DataSourceRM(**cfg)  # 2. config from arguments - Standard
+
+    # Manual registration
+    src = DataSourceRM('SourceName', root=data_root)  # name positional - Special init!
+    assert str(src)
+    assert not src.is_listed
+    DataSourceRM._manager.add_resource(src)
+    assert src.is_listed
+    assert src == DataSourceRM('SourceName')  # Query config     # 4. config from query   - Special init!
+    assert src == DataSourceRM.from_config('SourceName')  # ?? Alternative - Dedicated method for query!
+
+    src = DataSourceRM(root=data_root)  # Optional name - default|special init (need?)
+    assert repr(src)
+
+
+@log_func
+def case_resource_discoveries():
+    # Resource Manager may be found also by a part of its name
+    sources = mm.get('source')
+    assert sources.discover()  # some has been registered
+
+    # Manager contains its associated Resource Model as attribute
+    res = sources.list('res')[0]
+    assert sources.model(res.name) == res
+
+    # There is a shortcut version of this sequence:
+    assert mm.discover('source') == len(sources)
+
+    # Or, to discover resources from all the currently registered models
+    mm.discover()
+
+
+@log_func
+def scenario_work_with_collection():
+    from iad.dataman import create_collection
+    mm.discover()
+    dc = create_collection('KITTI',
+                           query=dict(subset='train'))
+
+    mm.discover('scheme')
+    src = mm.get('source')
+    src.discover('/mnt/algo/DataSets/depth', only=False)
+    mm.discover('scheme')
+
+    dc = create_collection('MID14S', temp_cache=True)
+    mm.discover('collect')
+
+
+@log_func
+def case_create_data_caster():
+    from iad.dataman import create_caster
+    my_dataset = create_caster(source=EnvLoc.DATA / 'my_dataset',
+                               scheme='{width}_{height}_{color}_{material}.jpg',
+                               filters=dict(color='BLUE',
+                                            material=['steal', 'wood'],
+                                            width=float(10).__le__,
+                                            condition_square='height == width'))
+
+
+@log_func
+def scenario_typical_flow():
+    from iad.dataman import create_caster, create_collection
+    case_resource_discoveries()
+    eth3d = create_caster('ETH3D')
+    dc = create_collection(datasets=[eth3d])
+    dc = create_collection(datasets=['MIDS12', 'FT3D'])
+    dc = create_collection('SmallQualityEval')

ialdev_dataman-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["flit_core>=3.11,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "ialdev-dataman"
+version = "0.1.0"
+description = "iad.dataman — data casting, resource models, and dataset bridge"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+
+dependencies = [
+    "ialdev-core",
+    "ialdev-io",
+    "ialdev-img",
+    "ialdev-maths",
+    "pandas>=1.3.0",
+    "numpy>=1.20.0",
+    "pydantic>=2.0",
+    "pyyaml>=5.4.0",
+    "regex>=2021.0.0",
+    "tqdm>=4.60.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=3.0.0",
+]
+
+[tool.flit.module]
+name = "iad.dataman"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+
+[tool.black]
+line-length = 100
+target-version = ["py310", "py311", "py312"]
+include = '\.pyi?$'

ialdev_dataman-0.1.0/src/iad/dataman/__init__.py

@@ -0,0 +1,22 @@
+"""iad.dataman — datacast, resman, and datasets bridge (``ialdev-dataman`` distribution)."""
+
+from .models import DataSourceRM, SchemeRM, DatasetRM, CollectionRM, DSample, discover
+from .factories import create_caster, create_collection, create_sink
+from .datacast.collect import DataCollection, SinkRepo
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "DataSourceRM",
+    "SchemeRM",
+    "DatasetRM",
+    "CollectionRM",
+    "DSample",
+    "discover",
+    "create_caster",
+    "create_collection",
+    "create_sink",
+    "DataCollection",
+    "SinkRepo",
+]

ialdev_dataman-0.1.0/src/iad/dataman/datacast/__init__.py

@@ -0,0 +1,8 @@
+""" Data Sets access and management tools."""
+
+__all__ = ['DataCaster', 'CasterConfig', 'DataCollection', 'SinkRepo',
+           'Col', 'Fetchable', 'CollectTable', 'CollectSeries']
+
+from .transtools import Fetchable, CollectSeries, CollectTable, Col
+from .caster import DataCaster, CasterConfig
+from .collect import DataCollection, SinkRepo