thebundle 0.0.1__py3-none-any.whl

bundle/__init__.py

@@ -0,0 +1,38 @@
+# Copyright 2023 HorusElohim
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import time
+from datetime import datetime
+import logging
+import typing
+from pathlib import Path
+
+from ._version import version, version_tuple
+from .logs import setup_logging, LOGGING_LEVEL, LogEmoji
+
+LOGGER = setup_logging(log_level=LOGGING_LEVEL, to_json=True)
+
+from . import data
+from . import entity
+from . import tasks
+from . import process
+from . import tests
+
+LOGGER.debug("bundle loaded")

bundle/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.0.1'
+__version_tuple__ = version_tuple = (0, 0, 1)

bundle/data/README.md

@@ -0,0 +1,133 @@
+# data 
+
+The data subpackage in BUNDLE introduces advanced data handling capabilities, focusing on the Dataclass and JSONData classes. These classes enhance Python's standard data handling methods, providing efficient and effective ways to manage complex data structures.
+
+*data.py*
+---
+The data.py module define the class `Dataclass``.
+
+## Dataclass
+Dataclass extends the standard Python dataclass to facilitate easy conversions between dataclass instances and dictionaries, addressing the need for streamlined data handling in Python.
+
+### Features
+
+* Python class -> dict 
+* dict -> Python class
+
+
+### Definition 
+
+It's preferred to initialize always the class attribute by `bundle.data.field` with a *default constructor* as for the dataclasses. 
+
+```python
+import bundle
+
+# Define a custom data class
+class MyData(bundle.data.Dataclass):
+    name: str = bundle.data.field(default_constructor=str)
+    value: int = bundle.data.field(default_constructor=int)
+```
+
+__Hint__: if need to assign default arguments in the CTor then use a lambda
+
+```python
+str_example: str = bundle.data.field(default_constructor=lambda: str("Nice"))
+```
+
+### Instantiation
+
+The instantiation happen as a normal dataclass.
+
+### Dataclass()
+Using the constructor.
+
+
+```python
+my_data = MyData(name="Example", value=42)
+```
+
+### from_dict: dict -> Python class
+Using the class method from_dict. Convert the dict into the class. 
+
+```python
+new_data = MyData.from_dict(data_dict)
+```
+
+### as_dict: Python class -> dict 
+Covert class to dict.
+
+```python
+data_dict = my_data.as_dict()
+```
+
+
+*json.py*
+---
+The json.py module define the class `JSONData``.
+
+
+## JSONData 
+The JSONData class in the data subpackage of BUNDLE extends the capabilities of the Dataclass by enabling serialization to and from JSON. This class is ideal for applications requiring robust data interchange, configuration management, or data persistence. JSONData provides a seamless way to handle JSON serialization and deserialization, including validation with jsonschema. It is designed to make working with JSON data in Python more intuitive, efficient, and reliable.
+
+### Features
+* JSON Serialization: Convert Python objects into JSON format, supporting both simple and complex data structures.
+* JSON Deserialization: Reconstruct Python objects from JSON, preserving the data's structure and types.
+* JSON Schema Validation: Validate JSON data against predefined schemas to ensure data integrity and format consistency.
+* Custom Serialization: Handle non-JSON serializable objects using pickle, extending the range of data types that can be serialized.
+
+### Definition 
+As Dataclass.
+
+```python
+import bundle
+
+@bundle.data.dataclass
+class MyJsonData(bundle.data.JSONData):
+    name: str = "Default Name"
+    value: int = 0
+```
+
+### Instantiation
+As Dataclass.
+
+```python
+my_json_data = MyJsonData(name="Example", value=42)
+```
+
+### Serialization
+Convert a JSONData instance to a JSON string or file:
+
+```python
+# Serialize to JSON string
+json_str = my_json_data.as_json()
+print(json_str)
+
+# Serialize to JSON file
+json_file_path = "data.json"
+my_json_data.dump_json(json_file_path)
+```
+
+__Hint__: During the serialization any Python object will be serialized with pickle.
+
+### Deserialization
+Load a JSONData instance from a JSON string or file:
+
+```python
+# Load from JSON file
+loaded_data = MyJsonData.from_json(json_file_path)
+print(loaded_data)
+```
+
+
+### JSON Schema Validation
+Generate and validate against a JSON schema:
+
+```python
+# Generate JSON schema
+json_schema_path = "schema.json"
+MyJsonData.to_jsonschema(json_schema_path)
+# Validate the instance against the schema
+is_valid = my_json_data.is_valid_by_jsonschema(json_schema_path)
+print(f"Validation result: {is_valid}")
+```
+

bundle/data/__init__.py

@@ -0,0 +1,39 @@
+# Copyright 2023 HorusElohim
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from dataclasses import asdict, dataclass, field, fields
+from pathlib import Path
+
+from .. import logging, LogEmoji
+
+LOGGER = logging.getLogger(__name__)
+
+
+def check_file_exist(path: str | Path, not_exist_raise=False) -> Path:
+    if not isinstance(path, Path | str):
+        raise ValueError(f"{type(path)=}, instead of [Path | str]")
+    path = Path(path)
+    if not_exist_raise and not path.exists():
+        raise ValueError(f"{path=}: ❌ NOT EXIST")
+    return path
+
+
+from .data import Dataclass
+from .json import JSONData

bundle/data/data.py

@@ -0,0 +1,147 @@
+# Copyright 2023 HorusElohim
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from __future__ import annotations
+
+from typing import List
+from typing import Any, Dict, List, Type, TypeVar, Union
+from pprint import pformat
+
+from . import dataclass, field, asdict, LOGGER, LogEmoji
+
+
+@dataclass
+class Dataclass:
+    """
+    A utility wrapper around the standard Python dataclass to facilitate easy
+    conversions between dataclass instances and dictionaries.
+
+    Attributes:
+        dataclass: Reference to the dataclass decorator.
+        field: Helper function to define fields with default values or other attributes.
+    """
+
+    class_type: str = field(default_factory=str)
+    dataclass = dataclass
+    field = field
+
+    def __post_init__(self):
+        # Set the class_type attribute to the name of the class
+        # This will be set for every instance of Dataclass or its subclasses
+        self.class_type = self.__class__.__name__
+
+    @staticmethod
+    def _is_annotation_present_in_mro(cls, field_name):
+        for base_class in cls.__mro__:
+            if field_name in getattr(base_class, "__annotations__", {}):
+                return True
+        return False
+
+    @staticmethod
+    def _recursive_dataclass_from_dict(target_class: Dataclass, source_dict: Union[Dict, List]) -> Dataclass:
+        """
+        Recursively convert a dictionary to a dataclass instance.
+        """
+        try:
+            # Log the expected fields from annotations in the MRO and the actual keys from source_dict
+            expected_fields = set()
+            if not hasattr(target_class, "__mro__"):
+                raise RuntimeError(f"{target_class=} not a class!")
+            for base_class in target_class.__mro__:
+                expected_fields.update(getattr(base_class, "__annotations__", {}).keys())
+
+            actual_keys = set(source_dict.keys())
+            LOGGER.debug(f"Expected fields for {target_class} from MRO: {sorted(expected_fields)}")
+            LOGGER.debug(f"Actual keys in source_dict: {sorted(actual_keys)}")
+
+            initialized_fields = {}
+            for field in source_dict:
+                LOGGER.debug(f"working on field {field}")
+                if Dataclass._is_annotation_present_in_mro(target_class, field):
+                    field_type = next(
+                        (
+                            getattr(base_class, "__annotations__", {}).get(field)
+                            for base_class in target_class.__mro__
+                            if field in getattr(base_class, "__annotations__", {})
+                        ),
+                        None,
+                    )
+                    if isinstance(source_dict[field], dict):
+                        initialized_fields[field] = Dataclass._recursive_dataclass_from_dict(field_type, source_dict[field])
+                    else:
+                        initialized_fields[field] = source_dict[field]
+                else:
+                    # If the field is not in annotations of any base classes, log this information
+                    LOGGER.warning(
+                        f"Field '%s' is not expected according to the annotations in the MRO of '%s'",
+                        field,
+                        target_class.__name__,
+                    )
+
+            # Log any missing fields
+            missing_fields = expected_fields - actual_keys
+            if missing_fields:
+                msg = f"Missing fields in source_dict that are expected in MRO of target_class: {missing_fields}"
+                LOGGER.error(msg)
+                raise ValueError(msg)
+
+            return target_class(**initialized_fields)
+        except KeyError as e:
+            # Key is missing in the dictionary, log an error before raising an exception
+            LOGGER.error(f"Key missing in source_dict for the expected field in MRO of target_class: {e}")
+            raise ValueError(f"Key missing in source_dict for the expected field in MRO of target_class: {target_class}: {e}")
+
+    @classmethod
+    def from_dict(cls: Dataclass, d: Dict[str, Any]) -> Dataclass:
+        """
+        Create an instance of the dataclass from a dictionary.
+
+        Args:
+            d: The dictionary containing data to be converted to a dataclass instance.
+
+        Returns:
+            An instance of the Dataclass initialized with data from the dictionary.
+
+        Raises:
+            ValueError: If a required key is missing in the dictionary.
+            TypeError: If there's a type mismatch or unexpected structure in the dictionary.
+        """
+        data_class = Dataclass._recursive_dataclass_from_dict(cls, d)
+        LOGGER.debug(LogEmoji.success)
+        return data_class
+
+    def as_dict(self) -> Dict[str, Any]:
+        """
+        Convert the dataclass instance to a dictionary.
+
+        Returns:
+            A dictionary representation of the dataclass instance.
+        """
+        data_dict = asdict(self)
+        LOGGER.debug(LogEmoji.success)
+        return data_dict
+
+    def __str__(self) -> str:
+        class_header = f"--------\n{self.class_type}\n--------"
+        formatted_data = pformat(self.as_dict(), indent=4, width=80, depth=None)
+        return f"{class_header}\n{formatted_data}\n--------"
+
+    def __hash__(self) -> int:
+        return hash(self.as_dict())

bundle/data/json.py

@@ -0,0 +1,273 @@
+# Copyright 2023 HorusElohim
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from __future__ import annotations
+
+import json as js
+from pathlib import Path
+import logging
+import abc
+import jsonschema
+from typing import Any, Dict, Union, Type, Set, get_type_hints
+import pickle
+import traceback
+import dataclasses
+from . import Dataclass, dataclass, fields, check_file_exist, LogEmoji
+
+
+def is_json_serializable(value: Any) -> bool:
+    """Check if a value is JSON serializable."""
+    try:
+        js.dumps(value, cls=js.JSONEncoder)
+        return True
+    except (TypeError, OverflowError):
+        return False
+
+
+LOGGER = logging.getLogger(__name__)
+
+DEFAULT_SCHEMA = "http://json-schema.org/draft-07/schema#"
+
+
+class CustomJSONEncoder(js.JSONEncoder):
+    """Extended JSON encoder to handle special data types using pickle for non-JSON serializable objects."""
+
+    def default(self, obj: Any) -> Any:
+        # Convert custom dataclass objects to dictionaries
+        LOGGER.debug(f"CustomJSONEncoder for {type(obj)}")
+        if isinstance(obj, Dataclass):
+            return obj.as_dict()
+        if isinstance(obj, Path):
+            return {"_path_": str(obj)}
+        try:
+            # Try to serialize using the parent class method
+            return super().default(obj)
+        except TypeError:
+            # If the object is not JSON serializable, use pickle
+            pickled_data = pickle.dumps(obj).hex()
+            return {"_pickle_": pickled_data}
+
+
+class CustomJSONDecoder(js.JSONDecoder):
+    """Custom JSON decoder to handle pickled data types."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, dct: Dict) -> Any:
+        if dct.get("_pickle_"):
+            return pickle.loads(bytes.fromhex(dct["_pickle_"]))
+        if dct.get("_path_"):
+            return Path(dct["_path_"])
+        return dct
+
+
+class JSONDataABC(abc.ABC):
+    """Abstract base class representing objects that can be serialized to and from JSON."""
+
+    @abc.abstractclassmethod
+    def from_json(cls: Type[JSONData], path: Union[str, Path]) -> JSONData:
+        """Deserialize from a JSON file."""
+        pass
+
+    @abc.abstractmethod
+    def dump_json(self, path: Union[str, Path]) -> None:
+        """Serialize to a JSON file."""
+        pass
+
+    @property
+    @abc.abstractmethod
+    def json_encoder(self) -> CustomJSONEncoder:
+        """Get the JSON encoder for this class."""
+        pass
+
+
+@dataclass(unsafe_hash=True)
+class JSONData(JSONDataABC, Dataclass):
+    """A dataclass that can be serialized to and from JSON."""
+
+    json_decoder = CustomJSONDecoder
+    json_encoder = CustomJSONEncoder
+
+    @staticmethod
+    def load_json_file(json_path: Union[Path, str], decoder: js.JSONDecoder = CustomJSONDecoder) -> Union[dict, None]:
+        json_path = check_file_exist(json_path, not_exist_raise=True)
+        try:
+            with Path(json_path).open("r") as file:
+                obj_dict = js.load(file, cls=decoder)
+                LOGGER.debug(f"{json_path=} {LogEmoji.success}")
+                return obj_dict
+        except jsonschema.ValidationError:
+            LOGGER.error(f"{json_path=} {LogEmoji.failed} \n{traceback.format_exc()}")
+            return None
+
+    @staticmethod
+    def dump_json_file(
+        obj_dict: Dict,
+        path: Union[Path, str],
+        encoder: js.JSONEncoder = CustomJSONEncoder,
+    ) -> Union[dict, None]:
+        path = check_file_exist(path)
+        try:
+            with Path(path).open("w") as file:
+                js.dump(obj_dict, file, indent=4, cls=encoder)
+                LOGGER.debug(f"{path=} {LogEmoji.success}")
+                return obj_dict
+        except jsonschema.ValidationError:
+            LOGGER.error(f"{path=} {LogEmoji.failed} \n{traceback.format_exc()}")
+            return None
+
+    @classmethod
+    def from_json(cls: Type[JSONData], path: Union[str, Path]) -> JSONData:
+        """Load the dataclass from a JSON file."""
+        obj_dict = JSONData.load_json_file(json_path=path, decoder=cls.json_decoder)
+        LOGGER.debug(f"{path=}{LogEmoji.success}{obj_dict}")
+        return cls.from_dict(obj_dict)
+
+    def dump_json(self, path: Union[str, Path]) -> None:
+        """Save the dataclass to a JSON file."""
+        self.dump_json_file(obj_dict=self.as_dict(), path=path, encoder=self.json_encoder)
+        LOGGER.debug(f"{path=}{LogEmoji.success}")
+
+    def as_json(self) -> None:
+        """Convert the dataclass to a JSON string."""
+        return js.dumps(self.as_dict(), indent=4, cls=self.json_encoder)
+
+    @classmethod
+    def generate_jsonschema(
+        cls,
+        target_cls: Type[JSONData] = None,
+        processed_classes: Set[Type] = None,
+    ) -> Dict[str, Any]:
+        """Generate a JSON schema for this or a given dataclass."""
+        if processed_classes is None:
+            processed_classes = set()
+
+        target_cls = target_cls or cls
+        if target_cls in processed_classes:
+            return {"type": "object", "title": target_cls.__name__}
+
+        processed_classes.add(target_cls)
+
+        if not dataclasses.is_dataclass(target_cls):
+            raise TypeError(f"Expected a dataclass type, got {target_cls}")
+
+        type_to_schema = {
+            int: "integer",
+            float: "number",
+            str: "string",
+            bool: "boolean",
+            list: "array",
+            dict: "object",
+        }
+
+        def get_pickled_schema() -> Dict[str, Any]:
+            return {
+                "type": "object",
+                "properties": {
+                    "_pickle_": {"type": "boolean", "const": True},
+                    "data": {"type": "string"},
+                },
+                "required": ["_pickle_", "data"],
+            }
+
+        def get_schema_for_type(type_hint: Type) -> Dict[str, Any]:
+            LOGGER.debug(f"Working on {type_hint=}")
+            if type_hint in type_to_schema:
+                return {"type": type_to_schema[type_hint]}
+            elif getattr(type_hint, "__origin__", None) == list:
+                item_type = type_hint.__args__[0]
+                return {"type": "array", "items": get_schema_for_type(item_type)}
+            elif dataclasses.is_dataclass(type_hint):
+                LOGGER.debug(f"Recursive call on dataclass")
+                return cls.generate_jsonschema(type_hint, processed_classes)
+            else:
+                # Fallback for other non-dataclass types
+                return get_pickled_schema()
+
+        properties = {key: get_schema_for_type(type_hint) for key, type_hint in get_type_hints(target_cls).items()}
+
+        required_fields = [f.name for f in fields(target_cls)]
+
+        return {
+            "$schema": DEFAULT_SCHEMA,
+            "title": target_cls.__name__,
+            "type": "object",
+            "properties": properties,
+            "required": required_fields,
+        }
+
+    @classmethod
+    def to_jsonschema(cls, path: Union[str, Path]):
+        """Save a JSON schema for this dataclass."""
+
+        report = f"to_schema({path})"
+        try:
+            with Path(path).open("w") as file:
+                js.dump(cls.generate_jsonschema(), file, indent=4)
+                LOGGER.debug(f"{report}{LogEmoji.success} ")
+        except Exception:
+            LOGGER.error(f"{report} {LogEmoji.failed} \n{traceback.format_exc()}")
+
+    @classmethod
+    def schema_validation(
+        cls: Type[JSONData],
+        json_path: Union[Path, str, None] = None,
+        jsonschema_path: Union[Path, str, None] = None,
+    ) -> bool:
+        """Validate JSON data against the schema."""
+        # Load the data from the provided path
+        report = f"schema_validation{json_path=}"
+        if json_path:
+            with Path(json_path).open("r") as file:
+                data = js.load(file)
+        else:
+            data = cls.as_dict()
+
+        # Get the schema for the current dataclass
+        if jsonschema_path:
+            with Path(jsonschema_path).open("r") as file:
+                schema = js.load(file)
+        else:
+            schema = cls.generate_jsonschema()
+
+        # Validate the data against the schema
+        try:
+            jsonschema.validate(data, schema)
+            LOGGER.debug(f"{report} {LogEmoji.success}")
+            return True
+        except jsonschema.ValidationError:
+            LOGGER.error(f"{report} {LogEmoji.failed} \n{traceback.format_exc()}")
+            return False
+
+    def is_valid_by_jsonschema(self, jsonschema_path: Union[Path, str, None] = None) -> bool:
+        """Validate JSON data against the schema."""
+        try:
+            # Load the data from the provided path
+            data = self.as_dict()
+            # Get the schema for the current dataclass
+            jsonschema_dict = JSONData.load_json_file(jsonschema_path) if jsonschema_path else self.generate_jsonschema()
+            # Validate the data against the schema
+            jsonschema.validate(data, jsonschema_dict)
+            LOGGER.debug(f"{jsonschema_path=} {LogEmoji.success}")
+            return True
+        except jsonschema.ValidationError:
+            LOGGER.error(f"{jsonschema_path=} {LogEmoji.failed} \n{traceback.format_exc()}")
+            return False