fmtr.tools 0.1__tar.gz

fmtr.tools-0.1/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.1
+Name: fmtr.tools
+Version: 0.1
+Summary: Frontmatter tools for AI projects
+Home-page: https://github.com/fmtr/fmtr.tools
+Author: Frontmatter
+Author-email: innovative.fowler@mask.pro.fmtr.dev
+License: Copyright © 2024 Frontmatter. All rights reserved.
+Platform: UNKNOWN
+Provides-Extra: augmentation
+Provides-Extra: docker.api
+Provides-Extra: logging
+Provides-Extra: parallel
+Provides-Extra: profiling
+Provides-Extra: test
+Provides-Extra: tokenization
+Provides-Extra: yaml
+
+UNKNOWN
+

fmtr.tools-0.1/fmtr/tools/augmentation_tools.py

@@ -0,0 +1,10 @@
+try:
+    from faker import Faker
+    import sre_yield
+except ImportError as exception:  # pragma: no cover
+    from fmtr.tools.tools import raise_missing_extra
+
+    raise_missing_extra('augmentation', exception)
+
+fake = Faker()
+to_generator = sre_yield.AllStrings

fmtr.tools-0.1/fmtr/tools/config.py

@@ -0,0 +1,13 @@
+from datetime import datetime
+
+from fmtr.tools.config_tools import ConfigClass
+
+
+class ToolsConfig(ConfigClass):
+    ENCODING = 'UTF-8'
+    LIBRARY_NAME = 'fmtr.tools'
+    DATE_FILENAME_FORMAT = '%Y-%m-%d'
+    DATETIME_FILENAME_FORMAT = f'{DATE_FILENAME_FORMAT}@%H-%M-%S'
+    DATETIME_NOW = datetime.utcnow()
+    DATETIME_NOW_STR = DATETIME_NOW.strftime(DATETIME_FILENAME_FORMAT)
+    SERIALIZATION_INDENT = 4

fmtr.tools-0.1/fmtr/tools/config_tools.py

@@ -0,0 +1,54 @@
+"""
+
+Config tools for downstream projects
+
+"""
+from dataclasses import dataclass, fields, Field as DataclassField
+
+from typing import List, Type
+
+
+class ConfigClass:
+    """
+
+    Base class for all config classes.
+
+    """
+
+    @classmethod
+    def process_field(cls, field):
+        """
+
+        Post-process field
+
+        """
+
+    @classmethod
+    def process_fields(cls):
+        """
+
+        Post-process fields
+
+        """
+        for field in cls.get_fields():
+            cls.process_field(field)
+
+    @classmethod
+    def get_fields(cls) -> List[DataclassField]:
+        """
+
+        Return fields
+
+        """
+        return fields(cls)
+
+    def __init_subclass__(cls, **kwargs):
+        """
+
+        Decorate subclasses as dataclasses
+
+        """
+        return dataclass(cls)
+
+
+Field = Type

fmtr.tools-0.1/fmtr/tools/dataclass_tools.py

@@ -0,0 +1,53 @@
+from dataclasses import fields as get_fields_tuple
+
+from fmtr.tools.tools import EMPTY
+
+
+def get_fields(cls, **filters) -> dict:
+    """
+
+    Get a dictionary of fields from a dataclass.
+
+    """
+    fields = {}
+    for field in get_fields_tuple(cls):
+        if all([getattr(field, key) is value for key, value in filters.items()]):
+            fields[field.name] = field
+
+    return fields
+
+
+def get_metadata(cls, **filters) -> dict:
+    """
+
+    Get a dictionary of fields metadata from a dataclass.
+
+    """
+    fields = get_fields(cls, **filters)
+    metadata = {name: field.metadata for name, field in fields.items()}
+    return metadata
+
+
+def get_enabled_fields(cls, name, enabled=True, default=EMPTY, **filters):
+    """
+
+    Get a dictionary of fields metadata from a dataclass filtered by enabled fields.
+
+    """
+    metadata = get_metadata(cls, **filters)
+
+    names = []
+
+    if isinstance(name, dict):
+        name = next(iter(name.keys()))
+
+    for key, field_meta in metadata.items():
+
+        if default is EMPTY:
+            value = field_meta[name]
+        else:
+            value = field_meta.get(name, default)
+        if value is enabled:
+            names.append(key)
+
+    return names

fmtr.tools-0.1/fmtr/tools/datatype_tools.py

@@ -0,0 +1,46 @@
+from typing import Any
+
+from distutils.util import strtobool
+
+from fmtr.tools.tools import Raise
+
+
+class TypeConversionFailed(ValueError):
+    """
+
+    Exception to raise for type conversion failure.
+
+    """
+
+
+def get_failure_message(raw, type_type):
+    """
+
+    Create generic type conversion failure message.
+
+    """
+    return f'Failed to convert "{raw}" (type: {type(raw)}) to type {type_type}'
+
+
+def to_bool(raw: Any, default=None) -> bool:
+    """
+
+    Convert a value to a Boolean
+
+    """
+
+    try:
+        converted = str(raw)
+        converted = strtobool(converted)
+        converted = bool(converted)
+        return converted
+    except ValueError as exception:
+        if default is Raise:
+            msg = get_failure_message(raw, bool)
+            raise TypeConversionFailed(msg) from exception
+        else:
+            return default
+
+
+def is_none(value: Any) -> bool:
+    return value is None

fmtr.tools-0.1/fmtr/tools/docker_tools.py

@@ -0,0 +1,36 @@
+import contextlib
+
+try:
+    import docker
+except ImportError as exception:  # pragma: no cover
+    from fmtr.tools.tools import raise_missing_extra
+
+    raise_missing_extra('docker', exception)
+
+
+@contextlib.contextmanager
+def Container(image, ports=None, name=None, **kwargs):
+    """
+
+    Run a Docker container in a context manager
+
+    """
+    client = docker.from_env()
+
+    try:
+        container = client.containers.get(name)
+        container.stop()
+        container.remove()
+    except docker.errors.NotFound:
+        pass
+
+    ports = {f'{port}/tcp': port for port in ports}
+    container = client.containers.run(image, ports=ports, detach=True, name=name, **kwargs)
+
+    try:
+        yield container
+    finally:
+        container.stop()
+        container.remove()
+
+

fmtr.tools-0.1/fmtr/tools/environment_tools.py

@@ -0,0 +1,76 @@
+"""
+
+Tools for handling environment variables etc.
+
+"""
+import os
+from collections.abc import Callable
+from datetime import date, datetime
+from typing import Any, Dict
+
+from fmtr.tools.datatype_tools import to_bool
+from fmtr.tools.path_tools import Path
+from fmtr.tools.tools import identity, EMPTY
+
+
+class MissingEnvironmentVariable(KeyError):
+    """
+
+    Exception for when a required environment variable is missing.
+
+    """
+
+
+def get_env_dict() -> Dict[str, str]:
+    """
+
+    Return environment variables as a standard dictionary.
+
+    """
+    environment_dict = dict(os.environ)
+    return environment_dict
+
+
+def get_env(name: str, default: Any = EMPTY, converter: Callable = identity, convert_empty: bool = False) -> Any:
+    """
+
+    Return the specified environment variable, handling default substitution and simple type conversion.
+
+    """
+    value = os.getenv(name, default)
+
+    if value is EMPTY:
+        msg = f'Environment variable "{name}" is required but has not been set'
+        raise MissingEnvironmentVariable(msg)
+
+    if value is not None or convert_empty:
+        value = converter(value)
+
+    return value
+
+
+def get_env_getter(converter: Callable) -> Callable:
+    """
+
+    Return an environment getter for the specified type.
+
+    """
+
+    def func(name: str, default: Any = EMPTY):
+        """
+
+        Environment getter that converts to the specified type
+
+        """
+        value = get_env(name, default=default, converter=converter)
+        return value
+
+    return func
+
+
+get_env_int = get_env_getter(lambda n: int(float(n)))
+get_env_float = get_env_getter(float)
+get_env_bool = get_env_getter(to_bool)
+get_env_date = get_env_getter(date.fromisoformat)
+get_env_datetime = get_env_getter(datetime.fromisoformat)
+get_env_path = get_env_getter(Path)

fmtr.tools-0.1/fmtr/tools/function_tools.py

@@ -0,0 +1,30 @@
+def combine_args_kwargs(args: dict=None, kwargs: dict=None) -> dict:
+    """
+
+    Combines arguments and keyword arguments into a single dictionary.
+
+    """
+    args = args or []
+    kwargs = kwargs or {}
+    args = {i: arg for i, arg in enumerate(args)}
+    args.update(kwargs)
+    if all(isinstance(key, int) for key in args.keys()):
+        args = list(args.values())
+    return args
+
+
+def split_args_kwargs(args_kwargs: dict) -> Tuple[list, dict]:
+    """
+
+    Splits arguments and keyword arguments into a list and a dictionary.
+
+    """
+    if isinstance(args_kwargs, list):
+        args, kwargs = args_kwargs, {}
+    else:
+        args = [arg for key, arg in args_kwargs.items() if isinstance(key, int)]
+        kwargs = {key: arg for key, arg in args_kwargs.items() if not isinstance(key, int)}
+
+    return args, kwargs
+
+

fmtr.tools-0.1/fmtr/tools/hash_tools.py

@@ -0,0 +1,13 @@
+from zlib import crc32
+
+from fmtr.tools.config import ToolsConfig
+
+
+def hash_unit(value: str) -> float:
+    """
+
+    Hash the input string to a value between 0.0 and 1.0 (not secure).
+
+    """
+    value = str(value).encode(ToolsConfig.ENCODING)
+    return float(crc32(value) & 0xffffffff) / 2 ** 32

fmtr.tools-0.1/fmtr/tools/iterator_tools.py

@@ -0,0 +1,45 @@
+from itertools import chain
+from typing import List, Dict, Any
+
+
+def enlist(value) -> List[Any]:
+    """
+
+    Make a non-list into a singleton list
+
+    """
+    enlisted = value if isinstance(value, list) else [value]
+    return enlisted
+
+
+def dict_records_to_lists(data: List[Dict[Any, Any]], missing: Any = None) -> Dict[Any, List[Any]]:
+    """
+
+    Convert a list of dictionaries to lists format
+
+    """
+    keys = set(chain.from_iterable([datum.keys() for datum in data]))
+    as_lists = {key: [] for key in keys}
+    for datum in data:
+        for key in keys:
+            as_lists[key].append(datum.get(key, missing))
+    return as_lists
+
+
+def get_batch_sizes(total, num_batches):
+    """
+
+    Calculate the sizes of batches for a given total number of items and number of batches.
+
+    """
+    return [total // num_batches + (1 if x < total % num_batches else 0) for x in range(num_batches)]
+
+
+def chunk_data(data, size: int):
+    """
+
+    Chunk data into batches of a given size, plus any remainder
+
+    """
+    chunked = [data[offset:offset + size] for offset in range(0, len(data), size)]
+    return chunked

fmtr.tools-0.1/fmtr/tools/json_tools.py

@@ -0,0 +1,23 @@
+import json
+
+from fmtr.tools.config import ToolsConfig
+
+
+def to_json(obj):
+    """
+
+    Serialise to JSON
+
+    """
+    json_str = json.dumps(obj, indent=ToolsConfig.SERIALIZATION_INDENT, ensure_ascii=False)
+    return json_str
+
+
+def from_json(json_str: str):
+    """
+
+    Deserialise from JSON
+
+    """
+    obj = json.loads(json_str)
+    return obj

fmtr.tools-0.1/fmtr/tools/logging_tools.py

@@ -0,0 +1,64 @@
+import sys
+
+try:
+    from loguru import logger as logger_loguru
+except ImportError as exception:  # pragma: no cover
+    from fmtr.tools.tools import raise_missing_extra
+
+    raise_missing_extra('logging', exception)
+
+from fmtr.tools.config import ToolsConfig
+from fmtr.tools.config_tools import ConfigClass
+from fmtr.tools.path_tools import Path
+from fmtr.tools.environment_tools import get_env
+
+
+class LoggingConfig(ConfigClass):
+    SEP = ' '
+    TIME = '<bold><green>{time:' + ToolsConfig.DATETIME_FILENAME_FORMAT + '}</green></bold>'
+    ICON = '<level>{level.icon}</level>'
+    LEVEL = '<level>{level:<8}</level>'
+    FILE = '{file}:{line}'
+    FUNCTION = '{function}(…)'
+    MESSAGE = '{message}'
+    DEFAULT_LEVEL_KEY = 'FMTR_LOG_LEVEL'
+    DEFAULT_LEVEL = get_env(DEFAULT_LEVEL_KEY, 'INFO')
+
+    FILENAME = f'log-{ToolsConfig.DATETIME_NOW_STR}.log'
+
+
+def default_filter(record):
+    return True
+
+
+def default_patch(record):
+    return record
+
+
+def get_logger(logger=logger_loguru, terminal=True, level=LoggingConfig.DEFAULT_LEVEL, time_format=LoggingConfig.TIME,
+               icon_format=LoggingConfig.ICON,
+               level_format=LoggingConfig.LEVEL, file_format=LoggingConfig.FILE, function_format=LoggingConfig.FUNCTION,
+               message_format=LoggingConfig.MESSAGE,
+               logfile=False, logfile_dir=None):
+    """
+
+
+
+    """
+    components = [time_format, icon_format, level_format, file_format, function_format, message_format]
+    format = LoggingConfig.SEP.join([component for component in components if component])
+    logger.remove()
+
+    if terminal:
+        logger.add(sys.stderr, format=format, level=level, filter=default_filter)
+        logger = logger.patch(default_patch)
+
+    if logfile:
+        logfile_dir = Path(logfile_dir or '.')
+        logfile_path = logfile_dir / LoggingConfig.FILENAME
+        logger.add(logfile_path, format=format)
+
+    return logger
+
+
+logger = get_logger()

fmtr.tools-0.1/fmtr/tools/parallel_tools.py

@@ -0,0 +1,103 @@
+from contextlib import nullcontext as NullContext
+from multiprocessing import cpu_count
+from typing import List, Callable, Any, Union
+
+from fmtr.tools.config_tools import ConfigClass
+from fmtr.tools.iterator_tools import dict_records_to_lists
+from fmtr.tools.logging_tools import logger
+from fmtr.tools.path_tools import Path
+
+try:
+    import dask
+    import dask.bag as db
+    from dask.diagnostics import ProgressBar
+except ImportError as exception:  # pragma: no cover
+    from fmtr.tools.tools import raise_missing_extra
+
+    raise_missing_extra('parallel', exception)
+
+
+class ParallelConfig(ConfigClass):
+    """
+
+    Configuration values.
+
+    """
+    THREADS = 'threads'
+    PROCESSES = 'processes'
+    SINGLE = 'single-threaded'
+
+
+def get_nullary_wrapper(func: Callable):
+    """
+
+    Dask cannot map a nullary function, as its argument list is empty. Hence this wrapper to force the function to take one dummy argument.
+
+    """
+
+    def wrap_nullary(dummy: None, **kwargs):
+        """
+
+        Ignore the dummy argument and run the function.
+
+        """
+        return func(**kwargs)
+
+    return wrap_nullary
+
+
+def apply_parallel(func: Callable, data: Union[List[Any], int], *args, num_workers: int = cpu_count(),
+                   scheduler: str = ParallelConfig.PROCESSES,
+                   parallelize: bool = True, show_progress: bool = False, return_future: bool = False, **kwargs) -> \
+List[Any]:
+    """
+
+    Helper function for a one-off, intensive parallel computation task.
+
+    """
+
+    if not parallelize and scheduler != ParallelConfig.SINGLE:
+        msg = f'Scheduler is set to "{scheduler}" but parallelization has been manually disabled.'
+        logger.warning(msg)
+        scheduler = ParallelConfig.SINGLE
+
+    data_kwargs = {}
+    if type(data) is int:  # If data is an integer, assume the function is nullary and just run it the specified number of times.
+        data_args = [[None] * data]
+        func = get_nullary_wrapper(func)
+    else:
+        data_args = []
+        is_data_lists = all(isinstance(datum, (tuple, list)) for datum in data)
+        is_data_dicts = all(isinstance(datum, dict) for datum in data)
+        if is_data_lists:  # If the data is a list of tuples/lists of arguments.
+            data_args += list(zip(*data))
+        elif is_data_dicts:  # If the data is a list of dictionaries of keyword arguments.
+            data_kwargs = dict_records_to_lists(data)
+        else:
+            data_args.append(data)  # Otherwise treat the data as a simple list of arguments.
+
+    dask.config.set({'temporary-directory': Path.temp()})
+
+    data_args = [db.from_sequence(value) for value in data_args]
+    data_kwargs = {key: db.from_sequence(values) for key, values in data_kwargs.items()}
+    future = db.map(func, *data_args, *args, **data_kwargs, **kwargs)
+
+    def get_results():
+        """
+
+        Function to compute results with the specified configuration.
+
+        """
+        if show_progress:
+            context = ProgressBar
+        else:
+            context = NullContext
+
+        with context():
+            return future.compute(scheduler=scheduler, num_workers=num_workers)
+
+    if return_future:  # Return a delayed function.
+        return get_results
+    else:
+        results = get_results()  # Compute and return results.
+        return results