toolchemy 0.2.185__py3-none-any.whl

toolchemy/utils/cacher/common.py

@@ -0,0 +1,174 @@
+import abc
+import hashlib
+import copy
+from abc import abstractmethod
+from typing import Any
+
+from toolchemy.utils.at_exit_collector import ICollectable, AtExitCollector
+from toolchemy.utils.datestimes import current_date_str, current_unix_timestamp
+
+
+class CacherInitializationError(Exception):
+    pass
+
+
+class CacheEntryDoesNotExistError(Exception):
+    pass
+
+
+class CacheEntryHasNotBeenSetError(Exception):
+    pass
+
+
+class ICacher(abc.ABC):
+    CACHER_MAIN_NAME = ".cache"
+
+    """
+    Cacher interface
+    """
+
+    @abc.abstractmethod
+    def sub_cacher(self, log_level: int | None = None, suffix: str | None = None) -> "ICacher":
+        pass
+
+    @abc.abstractmethod
+    def exists(self, name: str) -> bool:
+        """
+        Checks if there is a cache entry for a given name
+        """
+
+    @abc.abstractmethod
+    def set(self, name: str, content: Any, ttl_s: int | None = None):
+        """
+        Dumps a given object under a given cache entry name. The object must be pickleable.
+        """
+
+    @abc.abstractmethod
+    def unset(self, name: str):
+        """
+        Removes a cache entry for a given name
+        """
+
+    @abc.abstractmethod
+    def get(self, name: str) -> Any:
+        """
+        Loads an object for a given cache entry name. If it doesn't exist, an exception is thrown.
+        """
+
+    @property
+    @abstractmethod
+    def cache_location(self) -> str:
+        pass
+
+
+class BaseCacher(ICacher, ICollectable, abc.ABC):
+    def __init__(self):
+        self._name = self.__module__
+        self._cache_stats = {
+            "hit": 0,
+            "miss": 0,
+        }
+        AtExitCollector.register(self)
+
+    def collect(self) -> dict:
+        return self._cache_stats
+
+    def label(self) -> str:
+        return f"{self.__class__.__name__}({self._name})"
+
+    def exists(self, name: str) -> bool:
+        does_exist = self._exists(name)
+        if does_exist:
+            self._cache_stats["hit"] += 1
+        else:
+            self._cache_stats["miss"] += 1
+        return does_exist
+
+    @abc.abstractmethod
+    def _exists(self, name: str) -> bool:
+        """
+        Checks if there is a cache entry for a given name
+        """
+
+    def persist(self):
+        pass
+
+    @staticmethod
+    def hash(name: str) -> str:
+        hash_object = hashlib.md5(name.encode('utf-8'))
+        return hash_object.hexdigest()
+
+    @staticmethod
+    def create_cache_key(parts_plain: list | str | None = None, parts_hashed: list | str | None = None,
+                         with_current_date: bool = False) -> str:
+        replaceable_chars = "*.,'\"|<>[]?!-:;()@#$%^&{} "
+        if parts_plain is None and parts_hashed is None:
+            raise ValueError(f"You must provide the key components")
+        if parts_plain is None:
+            parts_plain = []
+        if parts_hashed is None:
+            parts_hashed = []
+        if isinstance(parts_plain, str):
+            parts_plain = [parts_plain]
+        if isinstance(parts_hashed, str):
+            parts_hashed = [parts_hashed]
+
+        for i, part_plain in enumerate(parts_plain):
+            for char_to_replace in list(replaceable_chars):
+                parts_plain[i] = str(parts_plain[i]).replace(char_to_replace, "_")
+
+        parts_hashed = [BaseCacher.hash(str(part_hashed)) for part_hashed in parts_hashed]
+        parts = parts_plain + parts_hashed
+        if with_current_date:
+            parts.append(current_date_str("%Y%m%d"))
+
+        return "_".join(parts)
+
+    def _envelop(self, content: Any, ttl_s: int | None = None) -> dict[str, Any]:
+        if not isinstance(content, dict) or ("data" not in content and "timestamp" not in content and "ttl_s" not in content):
+            entry_timestamp = current_unix_timestamp()
+            content = {'data': content, 'timestamp': entry_timestamp, 'ttl_s': ttl_s}
+        return content
+
+
+class DummyLock:
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class DummyCacher(BaseCacher):
+    def __init__(self, with_memory_store: bool = False):
+        super().__init__()
+        self._data = {}
+        self._with_memory_store = with_memory_store
+
+    @property
+    def cache_location(self) -> str:
+        return ""
+
+    def sub_cacher(self, log_level: int | None = None, suffix: str | None = None) -> "ICacher":
+        return DummyCacher(with_memory_store=self._with_memory_store)
+
+    def _exists(self, name: str) -> bool:
+        if not self._with_memory_store:
+            return False
+        return name in self._data
+
+    def set(self, name: str, content: Any, ttl_s: int | None = None):
+        if not self._with_memory_store:
+            return
+        self._data[name] = copy.deepcopy(content)
+
+    def unset(self, name: str):
+        if name in self._data:
+            del self._data[name]
+
+    def get(self, name: str) -> Any:
+        if not self._with_memory_store:
+            return None
+        if name not in self._data:
+            raise CacheEntryDoesNotExistError()
+        return self._data[name]

toolchemy/utils/datestimes.py

@@ -0,0 +1,77 @@
+import datetime
+import time
+from enum import Enum
+
+
+class Seconds(float, Enum):
+    NANOSECOND = 1 / 1000000000
+    MICROSECOND = 1 / 1000000
+    MILLISECOND = 1 / 1000
+    SECOND = 1
+    MINUTE = 60
+    HOUR = 60 * MINUTE
+    DAY = 24 * HOUR
+    WEEK = 7 * DAY
+    MONTH = 30 * DAY
+
+
+DEFAULT_DATE_FORMAT = "%d-%m-%Y"
+DEFAULT_DATETIME_FORMAT = "%d-%m-%Y %H:%M:%S"
+
+
+def format_str(datetime_str: str, date_format: str, target_datetime_format: str = DEFAULT_DATETIME_FORMAT) -> str:
+    datetime_obj = datetime.datetime.strptime(datetime_str, date_format)
+    return datetime_to_str(datetime_obj, datetime_format=target_datetime_format)
+
+
+def date_to_str(date_obj: datetime.date, date_format: str = DEFAULT_DATE_FORMAT) -> str:
+    return date_obj.strftime(date_format)
+
+
+def datetime_to_str(datetime_obj: datetime.datetime, datetime_format: str = DEFAULT_DATETIME_FORMAT) -> str:
+    return datetime_obj.strftime(datetime_format)
+
+
+def str_to_datetime(datetime_str: str, datetime_format: str = DEFAULT_DATETIME_FORMAT) -> datetime.datetime:
+    return datetime.datetime.strptime(datetime_str, datetime_format)
+
+
+def str_to_date(date_str: str, date_format: str = DEFAULT_DATE_FORMAT) -> datetime.date:
+    return datetime.datetime.strptime(date_str, date_format).date()
+
+
+def current_date_str(date_format: str = DEFAULT_DATE_FORMAT, time_delta_days: int | None = None) -> str:
+    date_ = datetime.date.today()
+    if time_delta_days is not None:
+        date_ -= datetime.timedelta(days=time_delta_days)
+    return date_to_str(date_, date_format)
+
+
+def current_datetime_str(datetime_format: str = DEFAULT_DATETIME_FORMAT, time_delta_days: int | None = None) -> str:
+    datetime_ = datetime.datetime.now()
+    if time_delta_days is not None:
+        datetime_ -= datetime.timedelta(days=time_delta_days)
+    return datetime_to_str(datetime_, datetime_format)
+
+
+def seconds_to_time_str(seconds: int | float) -> str:
+    seconds = int(seconds)
+    hours, remainder = divmod(seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return f"{hours:02}:{minutes:02}:{seconds:02}"
+
+
+def datetime_to_unix_timestamp(datetime_ob: datetime.datetime) -> int:
+    return int(datetime_ob.timestamp())
+
+
+def unix_timestamp_to_datetime(unix_timestamp: int) -> datetime.datetime:
+    return datetime.datetime.fromtimestamp(unix_timestamp)
+
+
+def str_to_unix_timestamp(datetime_str: str, datetime_format: str = DEFAULT_DATETIME_FORMAT) -> int:
+    return datetime_to_unix_timestamp(str_to_datetime(datetime_str, datetime_format))
+
+
+def current_unix_timestamp() -> int:
+    return int(time.time())

toolchemy/utils/locations.py

@@ -0,0 +1,111 @@
+import inspect
+import os
+import json
+from pathlib import Path
+
+
+class PathDoesNotExistError(Exception):
+    pass
+
+
+def _find_project_root(path: Path) -> Path:
+    for parent in [path] + list(path.parents):
+        if (parent / 'pyproject.toml').exists() or (parent / '.git').exists():
+            return parent
+    return path
+
+
+def get_external_caller_path(exclude_prefixes=None) -> str:
+    if exclude_prefixes is None:
+        exclude_prefixes = [str(Path(__file__).resolve().parent)]
+
+    for frame_info in inspect.stack():
+        module = inspect.getmodule(frame_info.frame)
+        if not module or not hasattr(module, '__file__'):
+            continue
+
+        path = Path(module.__file__).resolve()
+        if all(not str(path).startswith(prefix) for prefix in exclude_prefixes):
+            project_root_path = str(_find_project_root(path.parents[1]))
+            if "site-packages" in project_root_path:
+                return str(Path.cwd())
+            return project_root_path
+
+    if exclude_prefixes and len(exclude_prefixes) > 0:
+        return str(exclude_prefixes[0])
+    raise RuntimeError("Could not find external caller")
+
+
+class Locations:
+    def __init__(self, prefix_dirs: dict | None = None, root_path: str | None = None, objective_path_mode: bool = False) -> None:
+        if root_path is None:
+            root_path = get_external_caller_path()
+        self._dirs = {
+            "root": root_path.rstrip("/"),
+            "resources": os.path.join(root_path, "resources").rstrip("/"),
+            "data": os.path.join(root_path, "data").rstrip("/"),
+            "logs": os.path.join(root_path, "logs").rstrip("/"),
+        }
+        self._prefix_dirs = prefix_dirs
+        self._objective_path_mode = objective_path_mode
+
+    @property
+    def root(self) -> str:
+        ret = self._dirs["root"]
+        if self._objective_path_mode:
+            ret = Path(ret)
+        return ret
+
+    @staticmethod
+    def read_content(path: str | Path) -> str:
+        if isinstance(path, str):
+            path = Path(path)
+        return path.read_text()
+
+    @staticmethod
+    def read_json(path: str | Path) -> dict | list:
+        content = Locations.read_content(path)
+        try:
+            json_data = json.loads(content)
+        except json.JSONDecodeError as e:
+            raise json.JSONDecodeError(f"{e} (Failed to parse JSON from {path}: {e}. Parsed content:\n{content})", e.doc, e.pos) from e
+        return json_data
+
+    @staticmethod
+    def save_json(data: dict | list, path: str | Path):
+        with open(str(path), "w") as f:
+            json.dump(data, f, indent=4)
+
+    @staticmethod
+    def abs(path: str | Path) -> str:
+        return os.path.abspath(os.path.expanduser(str(path)))
+
+    def project_rel(self, path: str | Path) -> str | Path:
+        abs_path = str(self.abs(path))
+        rel_path = abs_path.replace(self.root, ".")
+        if self._objective_path_mode:
+            rel_path = Path(rel_path)
+        return rel_path
+
+    def in_root(self, elements: str | list[str] | None = None) -> str | Path:
+        return self.in_("root", elements)
+
+    def in_resources(self, elements: str | list[str] | None = None) -> str | Path:
+        return self.in_("resources", elements)
+
+    def in_data(self, elements: str | list[str] | None = None) -> str | Path:
+        return self.in_("data", elements)
+
+    def in_(self, base_dir: str, elements: str | list[str] | None = None) -> str | Path:
+        if isinstance(elements, str):
+            elements = [elements]
+        if elements is None:
+            elements = []
+        if base_dir not in self._dirs:
+            raise ValueError(f"There is no '{base_dir}' dir defined")
+        if self._prefix_dirs and base_dir in self._prefix_dirs:
+            elements = [self._prefix_dirs[base_dir]] + elements
+        ret = str(os.path.join(self._dirs[base_dir], *elements)).rstrip("/")
+        if self._objective_path_mode:
+            ret = Path(ret)
+        return ret

toolchemy/utils/logger.py

@@ -0,0 +1,76 @@
+import logging
+import pathlib
+from colorlog import ColoredFormatter
+
+from toolchemy.utils.utils import _caller_module_name
+
+
+def get_logger(name: str | None = None, level: int = logging.INFO, log_dir: str | None = None, with_time: bool = True,
+               with_module_name: bool = True, with_log_level: bool = True, short_module_name: bool = False, say_hi: bool = False) -> logging.Logger:
+    name = name or _caller_module_name()
+
+    datetime_format = "%H:%M:%S"
+    prompts_parts = []
+    if with_time:
+        prompts_parts.append('%(asctime)s')
+    if with_module_name:
+        module_format = '%(name)s'
+        if short_module_name:
+            module_format = '%(module)s'
+        prompts_parts.append(module_format)
+
+    if with_log_level:
+        prompts_parts.append('%(levelname)s')
+
+    msg_format = "%(log_color)s|" + " ".join(prompts_parts) + "|%(reset)s %(message)s"
+
+    formatter = ColoredFormatter(fmt=msg_format, datefmt=datetime_format, force_color=True, log_colors={
+		'DEBUG':    'cyan',
+		'INFO':     'green',
+		'WARNING':  'yellow',
+		'ERROR':    'red',
+		'CRITICAL': 'red,bg_white',
+	},
+	secondary_log_colors={})
+
+    logger = logging.getLogger(name)
+    for handler in (logger.handlers or []):
+        logger.removeHandler(handler)
+
+    logger.setLevel(level)
+
+    if not logger.handlers:
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+        if log_dir is not None:
+            log_dir_path = pathlib.Path(log_dir)
+            log_dir_path.mkdir(parents=True, exist_ok=True)
+
+            log_file = log_dir_path / f"{name.replace('.', '_')}.log"
+            file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8')
+            file_handler.setFormatter(formatter)
+            logger.addHandler(file_handler)
+
+    logger.propagate = False
+
+    if say_hi:
+        logger.info("Hi:)")
+        logger.debug("Debug mode ON")
+        logger.debug("All handlers:")
+        for handler in logger.handlers:
+            logger.debug(f"- name: {handler.get_name()}, format: {handler.formatter._fmt}, level: {handler.level}")
+
+    return logger
+
+
+def testing():
+    logger = get_logger("toolchemy.utils.logger", level=logging.DEBUG, say_hi=True)
+    logger.warning("Testing logger setup WARNING")
+    logger.info("Testing logger setup INFO")
+    logger.debug("Testing logger setup DEBUG")
+
+
+if __name__ == "__main__":
+    testing()

toolchemy/utils/timer.py

@@ -0,0 +1,23 @@
+import time
+
+
+class Timer:
+    def __init__(self, started: bool = True):
+        self._started = started
+        self._started_time = None
+        self._last_tap_time = None
+        self.reset()
+
+    def reset(self):
+        self._started_time = time.time()
+        self._last_tap_time = time.time()
+
+    def tap(self, since_last: bool = False) -> float:
+        now_time = time.time()
+        since_time = self._started_time
+        if since_last and self._last_tap_time:
+            since_time = self._last_tap_time
+
+        self._last_tap_time = now_time
+
+        return now_time - since_time

toolchemy/utils/utils.py

@@ -0,0 +1,168 @@
+import copy
+import datetime
+import inspect
+import json
+import numpy as np
+import random
+import torch
+import hashlib
+import base64
+
+from dataclasses import asdict, is_dataclass
+from typing import Any
+
+from toolchemy.utils.datestimes import datetime_to_str
+
+
+DEFAULT_SEED = 1337
+
+MEGABYTE = 1024 ** 2
+GIGABYTE = 1024 ** 3
+
+
+def seed_init_fn(x, only_deterministic: bool = False):
+    seed = DEFAULT_SEED + x
+    np.random.seed(seed)
+    random.seed(seed)
+    torch.manual_seed(seed)
+    if only_deterministic:
+        torch.use_deterministic_algorithms(True)
+
+
+def bytes_to_str(byte_data: bytes) -> str:
+    for encoding in ['utf-8', 'latin-1', 'ascii']:
+        try:
+            return byte_data.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    raise ValueError("Unknown encoding")
+
+
+def pp_cast(msg: Any, skip_fields: list | None = None) -> Any:
+    msg_copy = copy.deepcopy(msg)
+    if is_dataclass(msg_copy):
+        msg_copy = to_json(msg_copy)
+    if isinstance(msg_copy, bytes):
+        msg_copy = bytes_to_str(msg_copy)
+    if isinstance(msg_copy, dict):
+        for key in list(msg_copy):
+            if skip_fields and key in skip_fields:
+                del msg_copy[key]
+                continue
+            msg_copy[key] = pp_cast(msg_copy[key], skip_fields=skip_fields)
+    if isinstance(msg_copy, list):
+        for i, el in enumerate(msg_copy):
+            msg_copy[i] = pp_cast(el, skip_fields=skip_fields)
+    if isinstance(msg_copy, np.ndarray):
+        msg_copy = pp_cast(msg_copy.tolist(), skip_fields=skip_fields)
+    if isinstance(msg_copy, float):
+        msg_copy = ff(msg)
+    if isinstance(msg_copy, datetime.datetime):
+        msg_copy = datetime_to_str(msg_copy)
+    if isinstance(msg_copy, object) and type(msg_copy).__module__ != "builtins":
+        msg_copy = json.loads(json.dumps(msg_copy, default=vars))
+    return msg_copy
+
+
+def pp(msg: str | dict | float | int | list, skip_fields: list | None = None, print_msg: bool = True) -> str:
+    msg_ = pp_cast(msg, skip_fields=skip_fields)
+    if isinstance(msg_, dict):
+        msg_ = json.dumps(msg_, indent=4, ensure_ascii=False)
+    if isinstance(msg_, list):
+        if len(msg_) > 0 and isinstance(msg_[0], dict):
+            msg_ = json.dumps(msg_, indent=4, ensure_ascii=False)
+    if isinstance(msg_, int) or isinstance(msg_, float):
+        msg_ = ff(msg_)
+    if print_msg:
+        print(msg_)
+    return msg_
+
+
+
+def ff(fval: float | list[float] | int | list[int] | dict | str | np.float32, precision: int = 2):
+    if isinstance(fval, float):
+        return "%0.*f" % (precision, fval)
+    if isinstance(fval, int):
+        return str(fval)
+    if isinstance(fval, list):
+        return [ff(v, precision=precision) for v in fval]
+    if isinstance(fval, dict):
+        return {k: ff(v, precision=precision) for k, v in fval.items()}
+    if isinstance(fval, str):
+        return ff(float(fval), precision)
+    if isinstance(fval, np.float32):
+        return ff(fval.item(), precision)
+    raise ValueError(f"Unsupported type: {type(fval)}")
+
+
+def to_json(dataclass_container, key_prefix: str = None, exclude: list[str] | None = None) -> dict:
+    if exclude is None:
+        exclude = []
+    data_dict = asdict(dataclass_container)
+
+    parsed_dict = {}
+    for k, v in data_dict.items():
+        if k in exclude:
+            continue
+        new_key = k
+        if key_prefix is not None:
+            new_key = f"{key_prefix}_{k}"
+        if v is None:
+            v = ""
+        if isinstance(v, bool):
+            v = int(v)
+        if isinstance(v, list) and len(v) > 0 and isinstance(v[0], str):
+            v = ', '.join(v)
+        if is_dataclass(v):
+            v = to_json(v, key_prefix=key_prefix, exclude=exclude)
+        parsed_dict[new_key] = v
+
+    return parsed_dict
+
+
+def hash_dict(input_dict: dict) -> str:
+    json_str = json.dumps(input_dict, sort_keys=True)
+    json_bytes = json_str.encode('utf-8')
+    hash_bytes = hashlib.sha256(json_bytes).digest()
+    base64_hash = base64.b64encode(hash_bytes).decode('utf-8')
+
+    return base64_hash
+
+
+def normalize_path_str(path_: str) -> str:
+    return path_.\
+        replace("./", "_").\
+        replace("~/", ""). \
+        replace("~", ""). \
+        replace("/", "_").\
+        replace("-", "").\
+        replace(":", "_").\
+        replace("?", "_").\
+        replace("&", "_")
+
+
+def split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+    num_chunks = (len(text) - chunk_overlap) // (chunk_size - chunk_overlap)
+
+    chunks = []
+    for i in range(num_chunks):
+        start = i * (chunk_size - chunk_overlap)
+        end = start + chunk_size
+        chunks.append(text[start:end])
+
+    chunks.append(text[num_chunks * (chunk_size - chunk_overlap):])
+
+    return chunks
+
+
+def truncate(s: str, limit: int) -> str:
+    if len(s) <= limit:
+        return s
+    return f"{s[:limit]} (...{str(len(s) - limit)} more chars)"
+
+
+def _caller_module_name(offset: int = 2) -> str:
+    frame = inspect.stack()[offset]
+    module = inspect.getmodule(frame.frame)
+    namespace = module.__name__ if module and hasattr(module, '__name__') else '__main__'
+    return namespace

toolchemy/vision/__init__.py

@@ -0,0 +1,5 @@
+from .image import ImageProcessor
+from .caption_overlay import add_caption, Caption
+
+
+__all__ = ["ImageProcessor", "add_caption", "Caption"]