danielutils 1.1.28__py3-none-any.whl

danielutils/reflection/module/lazy_module.py

@@ -0,0 +1,104 @@
+"""
+Lazy module loading functionality.
+
+This module provides transparent lazy loading of optional modules,
+allowing applications to start even when optional dependencies are missing.
+The modules are only imported when actually used, providing clear error messages.
+"""
+
+import sys
+from typing import Any, Optional
+
+
+class LazyModule:
+    """
+    A proxy for modules that should only be imported when actually used.
+
+    This allows code to run even when optional dependencies are missing,
+    but provides clear error messages when the functionality is actually needed.
+    """
+
+    def __init__(self, module_name: str, error_message: str, install_hint: str):
+        self._module_name = module_name
+        self._module: Optional[Any] = None
+        self._imported = False
+
+        if not error_message:
+            error_message = f"Module '{module_name}' is not available"
+
+        if install_hint and install_hint.strip():
+            error_message += f". {install_hint}"
+
+        self._error_message = error_message
+
+        # Set essential module attributes to make it look like a real module
+        self.__name__ = module_name
+        self.__file__ = None
+        self.__package__ = None
+        self.__path__ = None
+        self.__spec__ = None
+
+    def _ensure_imported(self):
+        """Import the module if not already imported."""
+        if not self._imported:
+            # Check if there's already a real module in sys.modules
+            existing_module = sys.modules.get(self._module_name)
+            if existing_module is not self and existing_module is not None:
+                # There's already a real module, use it
+                self._module = existing_module
+                self._imported = True
+            else:
+                # Try to import the real module
+                try:
+                    # Temporarily remove ourselves from sys.modules to avoid recursion
+                    if existing_module is self:
+                        del sys.modules[self._module_name]
+
+                    self._module = __import__(self._module_name)
+                    self._imported = True
+
+                    # Restore ourselves to sys.modules
+                    sys.modules[self._module_name] = self
+                except ImportError:
+                    # If import fails, raise the custom error message
+                    raise ImportError(self._error_message)
+        return self._module
+
+    def __getattr__(self, name):
+        """Delegate attribute access to the actual module."""
+        # Try to import the module first
+        try:
+            module = self._ensure_imported()
+            return getattr(module, name)
+        except ImportError:
+            # If import fails, raise the custom error message
+            # This is runtime execution, not runtime loading
+            raise ImportError(self._error_message)
+
+    def __dir__(self):
+        """Support dir() on the lazy module."""
+        if self._imported:
+            return dir(self._module)
+        return []
+
+    @property
+    def module(self):
+        """Access to the original module if it exists."""
+        if self._imported and self._module is not None:
+            return self._module
+        return None
+
+    def __repr__(self):
+        if self._imported:
+            return repr(self._module)
+        return f"<LazyModule '{self._module_name}' (not imported)>"
+
+
+def lazy_import(module_name: str, error_message: str, install_hint: str):
+    """
+    Set up lazy loading for a module.
+    The module will be replaced in sys.modules with a LazyModule instance.
+    It will only be truly imported when an attribute is accessed.
+    """
+    sys.modules[module_name] = LazyModule(
+        module_name, error_message, install_hint)

danielutils/reflection/module/module_reflections.py

@@ -0,0 +1,34 @@
+import logging
+from typing import Any
+import importlib
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def dynamically_load(module_name: str, obj_name: str) -> Any:
+    """dynamically loads the module and returns the object from this file
+
+    Args:
+        module_name (str): name of python module, (typically a file name without extension)
+        obj_name (str): the name of the wanted object
+
+    Returns:
+        Any: the object
+    """
+    try:
+        module = importlib.import_module(module_name)
+        obj = getattr(module, obj_name)
+        logger.info("Successfully loaded object '%s' from module '%s'", obj_name, module_name)
+        return obj
+    except ImportError as e:
+        logger.error("Failed to import module '%s': %s", module_name, e)
+        raise
+    except AttributeError as e:
+        logger.error("Object '%s' not found in module '%s': %s", obj_name, module_name, e)
+        raise
+
+
+__all__ = [
+    "dynamically_load"
+]

danielutils/reflection/module/package_reflection.py

@@ -0,0 +1,161 @@
+import sys
+import os
+import pkgutil
+from pathlib import Path
+from collections import defaultdict
+from typing import Set as Set, List as List, Dict as Dict
+from ..interpreter import get_python_version
+
+if get_python_version() >= (3, 9):
+    from builtins import list as List, set as Set, dict as Dict
+
+
+def get_all_modules() -> Set[str]:
+    all_modules = set()
+
+    # Get built-in modules
+    builtin_modules = set(o.strip("_") for o in sys.builtin_module_names)
+
+    # Get modules from the Python Standard Library
+    stdlib_modules = set()
+    for module in pkgutil.iter_modules():
+        try:
+            # type:ignore
+            if not module.ispkg and hasattr(module.module_finder, 'path') and module.module_finder.path.startswith(sys.prefix):
+                stdlib_modules.add(module.name)
+        except AttributeError:
+            pass
+
+    # Combine built-in modules and modules from the Python Standard Library
+    all_modules.update(builtin_modules)
+    all_modules.update(stdlib_modules)
+
+    all_modules.update({
+        "platform", 'os', 'pathlib', 'subprocess', "inspect", "types", "dataclasses", "__future__", "ctypes", "pkgutil",
+        "logging", "importlib", "enum", "multiprocessing", "traceback", "re", "threading"
+    })
+    return all_modules
+
+
+ALL_MODULES: set = get_all_modules()
+
+
+def get_imports_helper(path: str):
+    from danielutils import file_exists
+    if not file_exists(path):
+        raise ValueError(f"Can't find file {path}")
+    is_in_multiline_comment = False
+    with open(path, "r", encoding="utf-8") as f:
+        for l in f.readlines():
+            l = l.strip()
+            if l.startswith("#"):
+                continue
+            if '"""' in l:
+                if l.count('"""') % 2 == 1:
+                    is_in_multiline_comment = not is_in_multiline_comment
+
+            if is_in_multiline_comment:
+                continue
+
+            if any([l.startswith("import "), l.startswith("from ")]):
+                yield l
+
+
+def resolve_relative(base_path: str, statement: str) -> str:
+    res = base_path
+    splits = statement.split(".")
+    for s in splits:
+        if s == "":
+            res = str(Path(res).parent)
+        else:
+            res = os.path.join(res, s)
+            break
+    from danielutils import is_directory
+    if not is_directory(res):
+        res = f"{res}.py"
+    return res
+
+
+def resolve_absolute(statement: str) -> str:
+    if statement in ALL_MODULES:
+        return statement
+    if "." in statement:
+        statement = statement.split(".")[0]
+        return resolve_absolute(statement)
+    return statement
+
+
+def resolve_path(base_path, statement: str) -> str:
+    og = statement
+    FROM = "from"
+    IMPORT = "import"
+    if FROM in statement:
+        statement = statement[statement.index(FROM) + len(FROM):].strip()
+        statement = statement[:statement.index(IMPORT)].strip()
+        if statement.startswith("."):
+            return resolve_relative(base_path, statement)
+    else:
+        statement = statement[statement.index(IMPORT) + len(IMPORT):].strip()
+    return resolve_absolute(statement)
+
+
+def normalize_path(path: str) -> str:
+    from danielutils import is_directory
+    if is_directory(path):
+        path = os.path.join(path, '__init__.py')
+    return str(Path(path).absolute())
+
+
+def get_imports(path: str) -> Dict[str, Set[str]]:
+    res: Dict[str, Set[str]] = defaultdict(set)
+    i = 0
+    path = normalize_path(path)
+    queue: List[str] = [path]
+    while i < len(queue):
+        cur = queue[i]
+        imports = list(get_imports_helper(cur))
+        paths = [resolve_path(cur, imp) for imp in imports]
+        for sub_path in paths:
+            if sub_path in res:
+                continue
+            from danielutils import file_exists, is_directory
+            if not (file_exists(sub_path) or is_directory(sub_path)):
+                res[cur].add(sub_path)
+            else:
+                if sub_path not in res:
+                    sub_path = normalize_path(sub_path)
+                    queue.append(sub_path)
+                    res[cur].add(sub_path)
+        i += 1
+    return res
+
+
+def create_dependency_graph(path: str) -> 'Graph[str]':
+    from danielutils import Graph, MultiNode
+    res = dict(get_imports(path))
+    g: Graph[str] = Graph()
+    dct: Dict[str, MultiNode[str]] = {}
+    for k, v in res.items():
+        for o in v:
+            dct[o] = dct.get(o, MultiNode(o))
+        n = MultiNode(k, [dct[o] for o in v])
+        g.add_node(n)
+
+    return g
+
+
+def get_dependencies(path: str, topological_sort: bool = True) -> List[str]:
+    from danielutils import Graph
+    g: Graph[str] = create_dependency_graph(path)
+
+    if topological_sort:
+        return [n.data for n in g.topological_sort()]
+
+    return [n.data for n in g]
+
+
+__all__ = [
+    "get_dependencies",
+    "create_dependency_graph",
+    'ALL_MODULES'
+]

danielutils/retry_executor/__init__.py

@@ -0,0 +1,3 @@
+from .backoff_strategies import *
+from .backoff_strategy import *
+from .retry_executor import *

danielutils/retry_executor/backoff_strategies/__init__.py

@@ -0,0 +1,6 @@
+from .no_backoff import *
+from .constant_backoff import *
+from .exponential_backoff import *
+from .linear_backoff import *
+from .multiplicative_backoff import *
+from .functional_backoff import *

danielutils/retry_executor/backoff_strategies/constant_backoff.py

@@ -0,0 +1,25 @@
+import logging
+from ..backoff_strategy import BackOffStrategy
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class ConstantBackOffStrategy(BackOffStrategy):
+    """
+    will always back off exactly the same amount of time
+
+    :param delay: The amount of milliseconds to sleep
+    """
+
+    def __init__(self, delay: float) -> None:
+        if not delay >= 0:
+            logger.error("Invalid delay value: %s - must be positive", delay)
+            raise ValueError("delay must be positive")
+        delay = float(delay)
+        super().__init__(lambda: delay)
+
+
+__all__ = [
+    "ConstantBackOffStrategy"
+]

danielutils/retry_executor/backoff_strategies/exponential_backoff.py

@@ -0,0 +1,26 @@
+import logging
+from ..backoff_strategy import BackOffStrategy
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExponentialBackOffStrategy(BackOffStrategy):
+    def __init__(self, initial: float) -> None:
+        if not initial >= 0:
+            logger.error("Invalid initial value: %s - must be positive", initial)
+            raise ValueError("initial must be positive")
+        attempt: int = 1
+
+        def inner() -> float:
+            nonlocal attempt
+            attempt += 1
+            backoff_time = initial ** (attempt - 1)
+            return backoff_time
+
+        super().__init__(inner)
+
+
+__all__ = [
+    "ExponentialBackOffStrategy"
+]

danielutils/retry_executor/backoff_strategies/functional_backoff.py

@@ -0,0 +1,29 @@
+import logging
+from typing import Callable
+
+from ..backoff_strategy import BackOffStrategy
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class FunctionalBackoffStrategy(BackOffStrategy):
+    def __init__(self, func: Callable[[int], float]) -> None:
+        logger.debug("Initializing FunctionalBackoffStrategy with function: %s", func.__name__ if hasattr(func, '__name__') else type(func).__name__)
+        attempt = 1
+        logger.debug("Setting up functional backoff with custom function")
+
+        def inner() -> float:
+            nonlocal attempt
+            attempt += 1
+            backoff_time = func(attempt - 1)
+            logger.debug("Functional backoff attempt %s: %sms (func(%s))", attempt, backoff_time, attempt - 1)
+            return backoff_time
+
+        super().__init__(inner)
+        logger.debug("FunctionalBackoffStrategy initialized successfully")
+
+
+__all__ = [
+    "FunctionalBackoffStrategy"
+]

danielutils/retry_executor/backoff_strategies/linear_backoff.py

@@ -0,0 +1,23 @@
+import logging
+from ..backoff_strategy import BackOffStrategy
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class LinerBackoffStrategy(BackOffStrategy):
+    def __init__(self, initial: float, additive_term: float) -> None:
+        attempt = 1
+
+        def inner() -> float:
+            nonlocal attempt
+            attempt += 1
+            backoff_time = initial + additive_term * (attempt - 1)
+            return backoff_time
+
+        super().__init__(inner)
+
+
+__all__ = [
+    'LinerBackoffStrategy'
+]

danielutils/retry_executor/backoff_strategies/multiplicative_backoff.py

@@ -0,0 +1,23 @@
+import logging
+from ..backoff_strategy import BackOffStrategy
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class MultiplicativeBackoff(BackOffStrategy):
+    def __init__(self, initial_backoff: float) -> None:
+        attempt = 1
+
+        def inner() -> float:
+            nonlocal attempt
+            attempt += 1
+            backoff_time = initial_backoff * (attempt - 1)
+            return backoff_time
+
+        super().__init__(inner)
+
+
+__all__ = [
+    'MultiplicativeBackoff'
+]

danielutils/retry_executor/backoff_strategies/no_backoff.py

@@ -0,0 +1,17 @@
+import logging
+from ..backoff_strategy import BackOffStrategy
+from ...logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class NoBackOffStrategy(BackOffStrategy):
+
+    def __init__(self) -> None:
+        logger.debug("Initializing NoBackOffStrategy (no delay)")
+        super().__init__(lambda: 0.0)
+
+
+__all__ = [
+    'NoBackOffStrategy',
+]

danielutils/retry_executor/backoff_strategy.py

@@ -0,0 +1,28 @@
+import logging
+from ..aliases import Supplier
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class BackOffStrategy:
+    """
+    A class to create a common abstraction for backoff strategies
+    """
+
+    def __init__(self, supp: Supplier[float]) -> None:
+        self._supp = supp
+
+    def get_backoff(self) -> float:
+        """
+        Get the backoff time in milliseconds.
+
+        :return: amount of milliseconds to sleep
+        """
+        backoff_time = self._supp()
+        return backoff_time
+
+
+__all__ = [
+    "BackOffStrategy"
+]

danielutils/retry_executor/retry_executor.py

@@ -0,0 +1,55 @@
+import logging
+import time
+from typing import Generic, TypeVar, Optional
+
+from ..aliases import Supplier, Consumer
+from .backoff_strategies import ConstantBackOffStrategy
+
+from .backoff_strategy import BackOffStrategy
+from ..logging_.utils import get_logger
+
+T = TypeVar("T")
+
+logger = get_logger(__name__)
+
+
+class RetryExecutor(Generic[T]):
+    def __init__(self, backoff_strategy: BackOffStrategy = ConstantBackOffStrategy(200)) -> None:
+        self._backoff_strategy = backoff_strategy
+        logger.info("RetryExecutor initialized with backoff strategy: %s", type(backoff_strategy).__name__)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is not None:
+            logger.warning("RetryExecutor context exited with exception: %s: %s", exc_type.__name__, exc_val)
+
+    def execute(self, supp: Supplier[T], max_retries: int = 5,
+                exception_callback: Optional[Consumer[Exception]] = None) -> Optional[T]:
+        logger.info("Starting retry execution with max_retries=%s", max_retries)
+        
+        for i in range(max_retries):
+            try:
+                result = supp()
+                logger.info("Execution succeeded on attempt %s", i + 1)
+                return result
+            except Exception as e:
+                logger.warning("Attempt %s failed with %s: %s", i + 1, type(e).__name__, e)
+                if exception_callback:
+                    exception_callback(e)
+
+            if i != max_retries - 1:
+                self._sleep()
+        
+        logger.error("All %s attempts failed", max_retries)
+        return None
+
+    def _sleep(self) -> None:
+        backoff_time = self._backoff_strategy.get_backoff() / 1000
+        time.sleep(backoff_time)
+
+
+__all__ = [
+    "RetryExecutor",
+]

danielutils/snippets/__init__.py

@@ -0,0 +1 @@
+from .try_get import *

danielutils/snippets/try_get.py

@@ -0,0 +1,28 @@
+import logging
+from typing import Callable, Any, Optional
+from ..logging_.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+def try_get(supplier: Callable[[], Any]) -> Optional[Any]:
+    """try to get a value from a function and return the value or return None on fail
+
+    Args:
+        supplier (Callable[[], Any]): supplier function
+
+    Returns:
+        Optional[Any]: return value
+    """
+    try:
+        result = supplier()
+        logger.debug("try_get succeeded for supplier: %s", supplier.__name__)
+        return result
+    except Exception as e:
+        logger.debug("try_get failed for supplier: %s with %s: %s", supplier.__name__, type(e).__name__, e)
+        return None
+
+
+__all__ = [
+    "try_get"
+]

danielutils/system/__init__.py

@@ -0,0 +1,3 @@
+from .independent import *
+from .windows import *
+from .layered_command import *