ddeutil-workflow 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

ddeutil/workflow/vendors/__dict.py

@@ -1,333 +0,0 @@
-"""
-Reference:
-    * https://github.com/LarsHill/metadict
-"""
-
-from __future__ import annotations
-
-import contextlib
-import copy
-import keyword
-import re
-import warnings
-from collections.abc import (
-    Iterable,
-    Iterator,
-    KeysView,
-    Mapping,
-    MutableMapping,
-)
-from re import Pattern
-from typing import (
-    Any,
-    Optional,
-    TypeVar,
-)
-
-from typing_extensions import Self
-
-
-def _warning(
-    message,
-    category=UserWarning,
-    filename="",
-    lineno=-1,
-    file=None,
-    line="",
-):
-    """Monkey patch `warnings` to show UserWarning without the line information
-    of warnings call.
-    """
-    msg = warnings.WarningMessage(
-        message, category, filename, lineno, file, line
-    )
-    print(f"{msg.category.__name__}: {msg.message}")
-
-
-warnings.showwarning = _warning
-
-KT = TypeVar("KT")
-VT = TypeVar("VT")
-
-# NOTE: regex to enforce python variable/attribute syntax
-ALLOWED_VAR_SYNTAX: Pattern = re.compile(r"[a-zA-Z_]\w*")
-
-
-def complies_variable_syntax(name: Any) -> bool:
-    """Checks whether a given object is a string which complies the python
-    variable syntax.
-    """
-    if not isinstance(name, str) or keyword.iskeyword(name):
-        return False
-    name_cleaned = "".join(re.findall(ALLOWED_VAR_SYNTAX, name))
-    return name_cleaned == name
-
-
-class MetaDict(MutableMapping[KT, VT], dict):
-    """Class that extends `dict` to access and assign keys via attribute dot
-    notation.
-
-    Examples:
-        >>> d = MetaDict({'foo': {'bar': [{'a': 1}, {'a': 2}]}})
-        >>> d.foo.bar[1].a
-        2
-        >>> d["foo"]["bar"][1]["a"]
-        2
-        >>> d.bar = 'demo'
-        >>> d.bar
-        'demo'
-
-        `MetaDict` inherits from MutableMapping to avoid overwriting all `dict`
-    methods. In addition, it inherits from `dict` to pass the quite common
-    `isinstance(obj, dict) check.
-
-        Also, inheriting from `dict` enables json encoding/decoding without a
-    custom encoder.
-    """
-
-    def __init__(self, *args, nested_assign: bool = False, **kwargs) -> None:
-        # NOTE: check that 'nested_assign' is  of type bool
-        if not isinstance(nested_assign, bool):
-            raise TypeError(
-                "Keyword argument 'nested_assign' must be an instance of "
-                "type 'bool'"
-            )
-
-        # NOTE: init internal attributes and data store
-        self.__dict__["_data"]: dict[KT, VT] = {}
-        self.__dict__["_nested_assign"] = nested_assign
-        self.__dict__["_parent"] = kwargs.pop("_parent", None)
-        self.__dict__["_key"] = kwargs.pop("_key", None)
-
-        # update state of data store
-        self.update(*args, **kwargs)
-
-        # call `dict` constructor with stored data to enable object encoding
-        # (e.g. `json.dumps()`) that relies on `dict`
-        dict.__init__(self, self._data)
-
-    def __len__(self) -> int:
-        return len(self._data)
-
-    def __iter__(self) -> Iterator[KT]:
-        return iter(self._data)
-
-    def __setitem__(self, key: KT, value: VT) -> None:
-        # show a warning if the assigned key or attribute is used internally
-        # (e.g `items`, `keys`, etc.)
-        try:
-            self.__getattribute__(key)
-            key_is_protected = True
-        except (AttributeError, TypeError):
-            key_is_protected = False
-        if key_is_protected:
-            warnings.warn(
-                f"'{self.__class__.__name__}' object uses '{key}' internally. "
-                f"'{key}' can only be accessed via `obj['{key}']`.",
-                stacklevel=2,
-            )
-
-        # set key recursively
-        self._data[key] = self._from_object(value)
-
-        # update parent when nested keys or attributes are assigned
-        parent = self.__dict__.pop("_parent", None)
-        key = self.__dict__.get("_key", None)
-        if parent is not None:
-            parent[key] = self._data
-
-    def __getitem__(self, key: KT) -> VT:
-        try:
-            value = self._data[key]
-        except KeyError:
-            if self.nested_assign:
-                return self.__missing__(key)
-            raise
-
-        return value
-
-    def __missing__(self, key: KT) -> Self:
-        return self.__class__(
-            _parent=self, _key=key, nested_assign=self._nested_assign
-        )
-
-    def __delitem__(self, key: KT) -> None:
-        del self._data[key]
-
-    def __setattr__(self, attr: str, val: VT) -> None:
-        self[attr] = val
-
-    def __getattr__(self, key: KT) -> VT:
-        try:
-            return self[key]
-        except KeyError:
-            raise AttributeError(
-                f"'{self.__class__.__name__}' object has no attribute '{key}'"
-            ) from None
-
-    def __delattr__(self, key: KT) -> None:
-        try:
-            del self[key]
-        except KeyError:
-            raise AttributeError(
-                f"'{self.__class__.__name__}' object has no attribute '{key}'"
-            ) from None
-
-    def __str__(self) -> str:
-        return str(self._data)
-
-    def __repr__(self) -> str:
-        return repr(self._data)
-
-    @staticmethod
-    def repack_args(cls: type, state: dict) -> MetaDict:
-        """Repack and rename keyword arguments stored in state before feeding
-        to class constructor
-        """
-        _data = state.pop("_data")
-        _nested_assign = state.pop("_nested_assign")
-        return cls(_data, nested_assign=_nested_assign, **state)
-
-    def __reduce__(self) -> tuple:
-        """Return state information for pickling."""
-        return MetaDict.repack_args, (self.__class__, self.__dict__)
-
-    def __dir__(self) -> Iterable[str]:
-        """Extend dir list with accessible dict keys (enables autocompletion
-        when using dot notation)
-        """
-        dict_keys = [
-            key for key in self._data.keys() if complies_variable_syntax(key)
-        ]
-        return dir(type(self)) + dict_keys
-
-    def copy(self) -> Self:
-        return self.__copy__()
-
-    def __copy__(self) -> Self:
-        cls = self.__class__
-        result = cls.__new__(cls)
-        result.__dict__.update(
-            {k: copy.copy(v) for k, v in self.__dict__.items()}
-        )
-        return result
-
-    @classmethod
-    def fromkeys(
-        cls,
-        iterable: Iterable[KT],
-        value: Optional[VT] = None,
-    ) -> Self:
-        """Constructor MetaDict form keys iterator.
-
-        Examples:
-            >>> def iter_keys() -> Iterable[str]:
-            ...     for i in range(3):
-            ...         yield f"k{i}"
-            >>> MetaDict.fromkeys(iterable=iter_keys())
-            {'k0': None, 'k1': None, 'k2': None}
-        """
-        return cls({key: value for key in iterable})
-
-    def to_dict(self) -> dict:
-        return MetaDict._to_object(self._data)
-
-    @staticmethod
-    def _to_object(obj: Any) -> Any:
-        """Recursively converts all nested MetaDicts to dicts."""
-
-        if isinstance(obj, (list, tuple, set)):
-            if MetaDict._contains_mapping(obj):
-                value = type(obj)(MetaDict._to_object(x) for x in obj)
-            else:
-                value = obj
-        elif isinstance(obj, Mapping):
-            value = {k: MetaDict._to_object(v) for k, v in obj.items()}
-        else:
-            value = obj
-
-        return value
-
-    def _from_object(self, obj: Any) -> Any:
-        """Recursively converts all nested dicts to MetaDicts."""
-
-        if isinstance(obj, (list, tuple, set)):
-            if MetaDict._contains_mapping(obj):
-                value = type(obj)(self._from_object(x) for x in obj)
-            else:
-                value = obj
-        elif isinstance(obj, MetaDict):
-            value = obj
-        elif isinstance(obj, Mapping):
-            value = self.__class__(
-                {k: self._from_object(v) for k, v in obj.items()},
-                nested_assign=self._nested_assign,
-            )
-        else:
-            value = obj
-
-        return value
-
-    def _set_nested_assignment(self, val: bool):
-        self.__dict__["_nested_assign"] = val
-        for value in self.values():
-            if isinstance(value, (list, tuple, set)):
-                for elem in value:
-                    if isinstance(elem, MetaDict):
-                        elem._set_nested_assignment(val)
-            elif isinstance(value, MetaDict):
-                value._set_nested_assignment(val)
-
-    def enable_nested_assignment(self):
-        self._set_nested_assignment(True)
-
-    def disable_nested_assignment(self):
-        self._set_nested_assignment(False)
-
-    @contextlib.contextmanager
-    def enabling_nested_assignment(self):
-        """Context manager which temporarily enables nested key/attribute
-        assignment.
-        """
-        nested_assign = self.nested_assign
-        if not nested_assign:
-            self.enable_nested_assignment()
-        try:
-            yield self
-        finally:
-            if not nested_assign:
-                self.disable_nested_assignment()
-
-    @property
-    def nested_assign(self):
-        return self._nested_assign
-
-    @staticmethod
-    def _contains_mapping(
-        iterable: Iterable, ignore: Optional[type] = None
-    ) -> bool:
-        """Recursively checks whether an Iterable contains an instance of
-        Mapping.
-        """
-        for x in iterable:
-            if isinstance(x, Mapping):
-                if ignore is None or not isinstance(x, ignore):
-                    return True
-            elif isinstance(x, (list, set, tuple)):
-                return MetaDict._contains_mapping(x, ignore)
-        return False
-
-    # NOTE: Add the following inherited methods from collections.abc.Mapping
-    # directly to make pycharm happy to checking.
-    # (removing an annoying warning for dict unpacking)
-    def __contains__(self, key):
-        try:
-            self[key]
-        except KeyError:
-            return False
-        else:
-            return True
-
-    def keys(self):
-        """D.keys() -> a set-like object providing a view on D's keys"""
-        return KeysView(self)

ddeutil/workflow/vendors/aws.py

@@ -1,185 +0,0 @@
-from typing import Any, Optional
-
-try:
-    import boto3
-    import botocore.exceptions
-except ImportError:
-    raise ImportError(
-        "Please install boto3 package if want to use boto wrapped object.\n\t\t"
-        "$ pip install boto3"
-    ) from None
-
-
-class WrapBoto3:
-    """Difference in boto3 between resource, client, and session
-    docs: https://stackoverflow.com/questions/42809096/
-        difference-in-boto3-between-resource-client-and-session
-
-    .. config::
-
-        ~/.aws/credentials
-
-        [my-user]
-        aws_access_key_id = AKIAxxx
-        aws_secret_access_key = xxx
-
-        [my-role]
-        source_profile = my-user
-        role_arn = arn:aws:iam::123456789012:role/the-role
-
-        ~/.aws/config
-
-        [profile my-role]
-        region = ap-southeast-2
-    """
-
-    def __init__(
-        self,
-        access_key_id: str,
-        secret_access_key: str,
-        region_name: Optional[str] = None,
-        *,
-        role_session_name: Optional[str] = None,
-        role_arn: Optional[str] = None,
-        mfa_serial: Optional[str] = None,
-    ):
-        self.access_key_id = access_key_id
-        self.secret_access_key = secret_access_key
-        self.region_name: str = region_name or "ap-southeast-1"
-
-        # Optional for session.
-        self.role_session_name: str = role_session_name or "AssumeRoleSession"
-        self.role_arn = role_arn
-        self.mfa_serial = mfa_serial
-
-        # Create credential
-        self.cred = self.make_cred()
-
-    def make_cred(self) -> dict[str, str]:
-        if self.role_arn is None:
-            return {
-                "AccessKeyId": self.access_key_id,
-                "SecretAccessKey": self.secret_access_key,
-            }
-        # NOTE: A low-level client representing AWS Security Token Service (STS)
-        # >>> sess = boto3.session.Session(
-        # ...   aws_access_key_id=ARN_ACCESS_KEY,
-        # ...   aws_secret_access_key=ARN_SECRET_KEY
-        # ... )
-        # >>> sts_client = sess.client('sts')
-        sts_client = boto3.client(
-            service_name="sts",
-            region_name=self.region_name,
-            aws_access_key_id=self.access_key_id,
-            aws_secret_access_key=self.secret_access_key,
-        )
-        mfa_optional: dict[str, str] = {}
-        if self.mfa_serial:
-            mfa_otp: str = str(input("Enter the MFA code: "))
-            mfa_optional = (
-                {"SerialNumber": self.mfa_serial, "TokenCode": mfa_otp},
-            )
-        assumed_role = sts_client.assume_role(
-            RoleArn=self.role_arn,
-            RoleSessionName=self.role_session_name,
-            DurationSeconds=3600,
-            **mfa_optional,
-        )
-        # NOTE: From the response that contains the assumed role, get the
-        # temporary credentials that can be used to make subsequent API calls
-        return assumed_role["Credentials"]
-
-    @property
-    def session(self):
-        """Can use by
-        ``s3 = self.session.client('s3')``
-        ``s3 = self.session.resource('s3')``
-        """
-        return boto3.session.Session(
-            aws_access_key_id=self.cred["AccessKeyId"],
-            aws_secret_access_key=self.cred["SecretAccessKey"],
-            aws_session_token=self.cred.get("SessionToken"),
-        )
-
-    @property
-    def s3(self):
-        return boto3.client(
-            service_name="s3",
-            region_name=self.region_name,
-            aws_access_key_id=self.cred["AccessKeyId"],
-            aws_secret_access_key=self.cred["SecretAccessKey"],
-            aws_session_token=self.cred.get("SessionToken"),
-        )
-
-    def list_objects(self, bucket: str, prefix: str):
-        objs: list[dict[str, Any]] = []
-        kwargs = {"Bucket": bucket, "Prefix": prefix}
-        while True:
-            resp = self.s3.list_objects_v2(**kwargs)
-            for obj in resp["Contents"]:
-                objs.append(obj)
-            try:
-                kwargs["ContinuationToken"] = resp["NextContinuationToken"]
-            except KeyError:
-                break
-        return objs
-
-    def paginate(
-        self,
-        bucket: str,
-        prefix: str,
-        *,
-        marker: Optional[str] = None,
-        search: Optional[str] = None,
-    ):
-        """
-        .. docs:
-            - https://boto3.amazonaws.com/v1/documentation/api/latest/
-                guide/paginators.html
-
-        .. search::
-            - "Contents[?Size > `100`][]"
-            - "Contents[?contains(LastModified, `'"2022-01-01"'`)]"
-            - "Contents[?LastModified>=`YYYY-MM-DD`].Key"
-            - "DeleteMarkers[?LastModified>=`2020-07-07T00:00:00`
-                && IsLatest==`true`].[Key,VersionId]"
-        """
-        paginator = self.s3.get_paginator("list_objects_v2")
-        page_iterator = paginator.paginate(
-            Bucket=bucket,
-            Prefix=prefix,
-            PaginationConfig={
-                # 'MaxItems': 10,
-                "PageSize": 10,
-                "StartingToken": marker,
-            },
-        )
-
-        for page in page_iterator:
-            print("# This is new page")
-            print("Contents Count:", len(page["Contents"]))
-            if "NextContinuationToken" in page.keys():
-                print(page["NextContinuationToken"])
-
-        # filtered_iterator = page_iterator.search("Contents[?Size > `100`][]")
-        # for key_data in filtered_iterator:
-        #     print(key_data)
-
-        # page_iterator = paginator.paginate(
-        #     Bucket=bucket,
-        #     Prefix=prefix,
-        #     PaginationConfig={
-        #         'MaxItems': 10,
-        #         'PageSize': 10,
-        #         'StartingToken': marker
-        #     }
-        # )
-
-    def exists(self, bucket: str, prefix: str) -> bool:
-        try:
-            self.s3.head_object(Bucket=bucket, Key=prefix)
-            return True
-        except botocore.exceptions.ClientError as err:
-            if err.response["Error"]["Code"]:
-                return False
-            raise

ddeutil/workflow/vendors/minio.py

@@ -1,11 +0,0 @@
-class WarpMinio:
-
-    def __init__(
-        self,
-        host: str,
-        access_key: str,
-        secret_access_key: str,
-    ):
-        self.host: str = host
-        self.access_key: str = access_key
-        self.secret_access_key: str = secret_access_key

ddeutil/workflow/vendors/pd.py

@@ -1,13 +0,0 @@
-class PandasCSV: ...
-
-
-class PandasJson: ...
-
-
-class PandasParq: ...
-
-
-class PandasDb: ...
-
-
-class PandasExcel: ...

ddeutil/workflow/vendors/pg.py

@@ -1,11 +0,0 @@
-# ------------------------------------------------------------------------------
-# Copyright (c) 2022 Korawich Anuttra. All rights reserved.
-# Licensed under the MIT License. See LICENSE in the project root for
-# license information.
-# ------------------------------------------------------------------------------
-from __future__ import annotations
-
-from .__dataset import TblDataset
-
-
-class PostgresTbl(TblDataset): ...