fmtr.tools 1.1.1__py3-none-any.whl → 1.4.37__py3-none-any.whl

fmtr/tools/caching_tools.py

@@ -1,6 +1,10 @@
+import cachetools
+from datetime import timedelta, datetime
 from diskcache import Cache
 
-from fmtr.tools import logger, Path
+from fmtr.tools.constants import Constants
+from fmtr.tools.logging_tools import logger
+from fmtr.tools.path_tools.path_tools import Path
 
 
 class Dump(dict):
@@ -31,9 +35,9 @@ class Disk(Cache):
             if not path.parent.exists():
                 raise FileNotFoundError(f"Directory {path.parent=} does not exist")
             if path and not path.exists():
-                logger.warning(f'Cache does not exist. Will be created. "{path=}"...')
+                logger.warning(f'Cache does not exist. Will be created. {str(path)=}...')
 
-            logger.info(f'Initializing Disk Cache at path "{path=}"...')
+            logger.info(f'Initializing Disk Cache {str(path)=}...')
 
         super().__init__(directory=str(path / self.ROOT_KEY), **settings)
 
@@ -95,7 +99,101 @@ class Disk(Cache):
         return self.dump()
 
 
+class TLRU(cachetools.TLRUCache):
+    """
+
+    Subclass to include logging and simplify global TTU
+
+    """
+    MASK_MAPPING = '{key} ' + Constants.ARROW + ' {value}'
+
+    def __init__(self, maxsize=1_024, timer=datetime.now, getsizeof=None, ttu_static=None, desc=None):
+        """
+
+        Add overridable TTU method
+
+        """
+        super().__init__(maxsize=maxsize, ttu=self.get_ttu, timer=timer, getsizeof=getsizeof)
+        self.ttu_static = ttu_static
+        self.desc = desc
+
+    @property
+    def cache_desc(self):
+        """
+
+        Friendly description of cache
+
+        """
+        desc = self.desc or self.__class__.__name__
+        return desc
+
+    def get_ttu(self, _key, value, now) -> float | timedelta:
+        """
+
+        Default implementation just adds on the static TTU
+
+        """
+        return now + self.ttu_static
+
+    def expire(self, time=None):
+        """
+
+        Log expiry
+
+        """
+        items = super().expire(time)
+        if not items:
+            return items
+
+        with logger.span(f'{self.desc} cache expiry {len(items)=}...'):
+            for key, value in items:
+                logger.debug(self.MASK_MAPPING.format(key=key, value=value))
+
+        return items
+
+    def popitem(self):
+        """
+
+        Log eviction
+
+        """
+        key, value = super().popitem()
+        logger.debug(f'{self.desc} cache eviction: {self.MASK_MAPPING.format(key=key, value=value)}')
+        return key, value
+
+    def dump(self):
+        """
+
+        Dump contents
+
+        """
+        data = Dump(self.items())
+        return data
+
+    @property
+    def data(self):
+        """
+
+        Dump as property
+
+        """
+        return self.dump()
+
+
+
 if __name__ == '__main__':
+    sec10 = timedelta(seconds=10)
+    c = TLRU(ttu_static=sec10, maxsize=2, desc='Test Data')
+    c['test'] = 'val'
+    c['test2'] = 'val2'
+    c['test3'] = 'val3'
+    c
+
+
+
+
+
+
     path_tmp_cache = Path.cwd().parent.parent / 'data' / 'cache'
     tc = Disk(path_tmp_cache)
 

fmtr/tools/constants.py

@@ -14,6 +14,15 @@ class Constants:
     DATETIME_NOW_STR = DATETIME_NOW.strftime(DATETIME_FILENAME_FORMAT)
     SERIALIZATION_INDENT = 4
 
+    ENV_NESTED_DELIMITER = '__'
+
+    ARROW = '→'
+    ARROW_RIGHT = ARROW
+    ARROW_SEP = f' {ARROW} '
+
+    ARROW_LEFT = '←'
+
+    FMTR_DEV_KEY = 'FMTR_DEV'
     FMTR_LOG_LEVEL_KEY = 'FMTR_LOG_LEVEL'
     FMTR_OBS_API_KEY_KEY = 'FMTR_OBS_API_KEY'
     FMTR_OBS_HOST = 'obs.sv.fmtr.dev'
@@ -27,3 +36,35 @@ class Constants:
 
     FMTR_AI_HOST_KEY = 'FMTR_URL_HOST'
     FMTR_AI_HOST_DEFAULT = 'ai.gex.fmtr.dev'
+
+    FMTR_DEV_HOST = 'ws.gex.fmtr.dev'
+
+    FMTR_DEV_INTERFACE_URL = f'https://{FMTR_DEV_HOST}/'
+    FMTR_DEV_INTERFACE_SUB_URL_MASK = f'https://{{sub}}.{FMTR_DEV_HOST}/'
+
+    FILENAME_CONFIG = 'settings.yaml'
+    DIR_NAME_REPO = 'repo'
+    DIR_NAME_DATA = 'data'
+    DIR_NAME_CACHE = 'cache'
+    DIR_NAME_ARTIFACT = 'artifact'
+    DIR_NAME_SOURCE = 'source'
+    FILENAME_VERSION = 'version'
+    DIR_NAME_HF = 'hf'
+
+    ENTRYPOINT = 'entrypoint'
+    ENTRYPOINTS_DIR = f'{ENTRYPOINT}s'
+    ENTRYPOINT_FILE = f'{ENTRYPOINT}.py'
+    SCRIPTS_DIR = 'scripts'
+
+    PACKAGE_EXCLUDE_DIRS = {SCRIPTS_DIR, 'data', 'build', 'dist', '.*', '*egg-info*'}
+    INIT_FILENAME = '__init__.py'
+    DOCS_DIR = 'docs'
+    DOCS_CONFIG_FILENAME = 'mkdocs.yml'
+
+    DEVELOPMENT = "development"
+    PRODUCTION = "production"
+
+    INFRA = 'infra'
+
+    PROMPT_NONE_SPECIFIED = '[None Specified]'
+    WEBHOOK_URL_NOTIFY_KEY = 'WEBHOOK_URL_NOTIFY'

fmtr/tools/context_tools.py

@@ -0,0 +1,23 @@
+from contextlib import contextmanager, ExitStack
+
+
+@contextmanager
+def null():
+    """
+
+    Null context manager.
+
+    """
+    yield
+
+
+@contextmanager
+def contexts(*contexts):
+    """
+
+    Tee context managers.
+
+    """
+    with ExitStack() as stack:
+        resources = [stack.enter_context(context) for context in contexts]
+        yield resources

fmtr/tools/data_modelling_tools.py

@@ -1,4 +1,135 @@
-from pydantic import BaseModel, RootModel, ConfigDict
+import inspect
+from functools import cached_property
+from typing import ClassVar, List, Any, Dict
+
+from pydantic import BaseModel
+from pydantic import RootModel, ConfigDict
+from pydantic.fields import FieldInfo
+from pydantic.json_schema import SkipJsonSchema
+from pydantic_core import PydanticUndefined, PydanticUndefinedType
+
+from fmtr.tools.datatype_tools import is_optional, none_else
+from fmtr.tools.iterator_tools import get_class_lookup
+from fmtr.tools.string_tools import camel_to_snake
+from fmtr.tools.tools import Auto, Required, Empty
+
+
+class Field(FieldInfo):
+    """
+
+    Allow DRYer field definitions, set annotation and defaults at the same time, easier field inheritance, etc.
+
+    """
+    NAME = Auto
+    ANNOTATION = Empty
+    DEFAULT = Auto
+    FILLS = None
+    DESCRIPTION = None
+    TITLE = Auto
+    SKIP_SCHEMA = False
+    CONFIG = None
+
+    def __init__(self, annotation=Empty, default=Empty, description=None, title=None, fills=None, skip_schema=None, **kwargs):
+        """
+
+        Infer default from type annotation, if enabled, use class/argument fills to create titles/descriptions, etc.
+
+        """
+
+        fills_super = getattr(super(), 'FILLS', None)
+        self.fills = (fills_super or {}) | (self.FILLS or {}) | (fills or {})
+
+        skip_schema = none_else(skip_schema, self.SKIP_SCHEMA)
+
+        self.annotation = self.ANNOTATION if annotation is Empty else annotation
+        if self.annotation is Empty:
+            raise ValueError("Annotation must be specified.")
+
+        if skip_schema:
+            self.annotation = SkipJsonSchema[self.annotation]
+
+        default = self.get_default_auto(default)
+        if default is Required:
+            default = PydanticUndefined
+
+        description = self.get_desc(description)
+        title = self.get_title_auto(title)
+        kwargs |= (self.CONFIG or {})
+
+        super().__init__(annotation=self.annotation, default=default, title=title, description=description, **kwargs)
+
+    @classmethod
+    def get_name_auto(cls) -> str:
+        """
+
+        Infer field name, if set to auto.
+
+        """
+        if cls.NAME is Auto:
+            return camel_to_snake(cls.__name__)
+        elif cls.NAME is None:
+            return cls.__name__
+
+        return cls.NAME
+
+    @cached_property
+    def fills(self) -> Dict[str, str]:
+        """
+
+        Get fills with filled title merged in
+
+        """
+
+        fills_super = getattr(super(), 'FILLS', None)
+
+        return (fills_super or {}) | (self.FILLS or {}) | dict(title=self.get_title_auto())
+
+    def get_default_auto(self, default) -> type[Any] | None | PydanticUndefinedType:
+        """
+
+        Infer default, if not specified.
+
+        """
+
+        if default is not Empty:
+            return default
+
+        if self.DEFAULT is not Auto:
+            return self.DEFAULT
+
+        if is_optional(self.annotation):
+            return None
+        else:
+            return Required
+
+    def get_title_auto(self, mask) -> str | None:
+        """
+
+        Get title from classname/mask
+
+        """
+
+        if not mask:
+            mask = self.__class__.__name__ if self.TITLE is Auto else self.TITLE
+
+        if mask:
+            return mask.format(**self.fills)
+
+        return None
+
+    def get_desc(self, mask) -> str | None:
+        """
+
+        Fill description mask, if specified
+
+        """
+
+        mask = mask or self.DESCRIPTION
+
+        if mask:
+            return mask.format(**self.fills)
+
+        return None
 
 
 def to_df(*objs, name_value='value'):
@@ -51,30 +182,102 @@ class MixinFromJson:
         return self
 
 
-class Base(BaseModel, MixinFromJson):
+class CliRunMixin:
+    """
+
+    Mixin only so that it can also be used with Pydantic Settings.
+
+    TODO Ideally, the run method would be defined on dm.Base and the settings base would just inherit like set.Base(BaseSettings, dm.Base), but this isn't yet tested/could break Fields.
+
+    """
+
+    def run(self):
+        """
+
+        Behaviour when run as a CLI command, i.e. via Pydantic Settings. Run any subcommands then exit.
+
+        """
+
+        from pydantic_settings import get_subcommand
+
+        command = get_subcommand(self, is_required=False, cli_exit_on_error=False)
+
+        if not command:
+            return
+
+        result = command.run()
+        if inspect.isawaitable(result):
+            import asyncio
+            result = asyncio.run(result)
+
+        raise SystemExit(result)
+
+
+class Base(BaseModel, MixinFromJson, CliRunMixin):
     """
 
-    Base model
+    Base model allowing model definition via a list of custom Field objects.
 
     """
+    FIELDS: ClassVar[List[Field] | Dict[str, Field]] = []
 
-    def to_df(self, name_value='value'):
+    def __init_subclass__(cls, **kwargs):
         """
 
-        DataFrame representation with Fields as rows.
+        Fetch aggregated fields metadata from the hierarchy and set annotations and FieldInfo objects in the class.
 
         """
+        super().__init_subclass__(**kwargs)
 
-        objs = []
-        for name in self.model_fields.keys():
-            val = getattr(self, name)
-            objs.append(val)
+        fields = {}
+        for base in reversed(cls.__mro__):
 
-        df = to_df(*objs, name_value=name_value)
-        df['id'] = list(self.model_fields.keys())
-        df = df.set_index('id', drop=True)
+            try:
+                raw = base.FIELDS
+            except AttributeError:
+                raw = {}
+
+            if isinstance(raw, dict):
+                fields |= raw
+            else:
+                fields |= get_class_lookup(*raw, name_function=lambda cls_field: cls_field.get_name_auto())
+
+        cls.FIELDS = fields
+
+        for name, field in fields.items():
+            if name in cls.__annotations__:
+                continue
+
+            if inspect.isclass(field):
+                field = field()
+
+            setattr(cls, name, field)
+            cls.__annotations__[name] = field.annotation
+
+    def to_df(self, **kwargs):
+        """
+
+        DataFrame representation of Data Model.
+
+        """
+        from fmtr.tools import tabular
+
+        data = self.model_dump(**kwargs)
+        df = tabular.pd.json_normalize(data, sep='_')
         return df
 
+    @classmethod
+    def to_df_empty(cls):
+        """
+
+        Empty DataFrame
+
+        """
+        from fmtr.tools import tabular
+
+        row = {name: None for name in cls.model_fields.keys()}
+        df = tabular.pd.DataFrame([row])
+        return df
 
 class Root(RootModel, MixinFromJson):
     """
@@ -83,11 +286,21 @@ class Root(RootModel, MixinFromJson):
 
     """
 
-    def to_df(self):
+    def to_df(self, **kwargs):
         """
 
         DataFrame representation with items as rows.
 
         """
 
-        return to_df(*self.items)
+        """
+
+        DataFrame representation of Data Model.
+
+        """
+        from fmtr.tools import tabular
+
+        data = [item.model_dump(**kwargs) for item in self.items]
+        dfs = [tabular.pd.json_normalize(datum, sep='_') for datum in data]
+        df = tabular.pd.concat(dfs, axis=tabular.CONCAT_VERTICALLY)
+        return df

fmtr/tools/database_tools/__init__.py

@@ -0,0 +1,6 @@
+from fmtr.tools.import_tools import MissingExtraMockModule
+
+try:
+    from fmtr.tools.database_tools import document
+except ModuleNotFoundError as exception:
+    document = MissingExtraMockModule('db.document', exception)

fmtr/tools/database_tools/document.py

@@ -0,0 +1,51 @@
+from functools import cached_property
+from typing import List
+
+import beanie
+from beanie.odm import actions
+from pymongo import AsyncMongoClient
+
+from fmtr.tools import data_modelling_tools
+from fmtr.tools.constants import Constants
+from fmtr.tools.logging_tools import logger
+
+ModifyEvents = [
+    actions.Insert,
+    actions.Replace,
+    actions.Save,
+    actions.SaveChanges,
+    actions.Update
+]
+
+
+class Document(beanie.Document, data_modelling_tools.Base):
+    """
+
+    Document stub.
+
+    """
+
+
+class Client:
+
+    def __init__(self, name, host=Constants.FMTR_DEV_HOST, port=27017, documents: List[beanie.Document] | None = None):
+        self.name = name
+        self.host = host
+        self.port = port
+        self.documents = documents
+
+        self.client = AsyncMongoClient(self.uri, tz_aware=True)
+        self.db = self.client[self.name]
+
+    @cached_property
+    def uri(self):
+        return f'mongodb://{self.host}:{self.port}'
+
+    async def connect(self):
+        """
+
+        Connect
+
+        """
+        with logger.span(f'Connecting to document database {self.uri=} {self.name=}'):
+            return await beanie.init_beanie(database=self.db, document_models=self.documents)

fmtr/tools/datatype_tools.py

@@ -1,8 +1,9 @@
-from typing import Any
+from types import UnionType, NoneType
+from typing import Any, get_origin, get_args, Union, Annotated
 
 from fmtr.tools.tools import Raise
 
-TRUES = {str(True).lower(), str(1), 'y', 'yes'}
+TRUES = {str(True).lower(), str(1), 'y', 'yes', 'on'}
 
 
 class TypeConversionFailed(ValueError):
@@ -84,3 +85,22 @@ def none_else(value: Any, default: Any) -> Any:
     if is_none(value):
         return default
     return value
+
+
+def is_optional(annotation) -> bool:
+    """
+
+    Is type/annotation optional? todo should be in typing_tools?
+
+    """
+    if annotation is None:
+        return True
+
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+
+    if origin is Annotated:
+        return is_optional(args[0])
+
+    is_opt = (origin is UnionType or origin is Union) and NoneType in args
+    return is_opt

fmtr/tools/datetime_tools.py

@@ -0,0 +1,12 @@
+from datetime import datetime, timezone
+
+MIN = datetime.min.replace(tzinfo=timezone.utc)
+MAX = datetime.max.replace(tzinfo=timezone.utc)
+
+def now() -> datetime:
+    """
+
+    Now UTC
+
+    """
+    return datetime.now(tz=timezone.utc)

fmtr/tools/debugging_tools.py

@@ -1,10 +1,44 @@
-import pydevd_pycharm
+import dataclasses
 
 from fmtr.tools import environment_tools as env
 from fmtr.tools.constants import Constants
 
 MASK = 'Starting debugger at tcp://{host}:{port}...'
 
+
+@dataclasses.dataclass
+class ShellDebug:
+    """
+
+    Debugging information for shell commands
+
+    """
+    command: str
+    out: str
+    err: str
+    status: int
+
+    # timestamp: datetime=dataclasses.field(default_factory=datetime.now)
+
+    @classmethod
+    def from_path(cls, path_str):
+        """
+
+        Get debug info from path
+
+        """
+        from fmtr.tools import Path
+        path = Path(path_str).absolute()
+        data = {field.name: (path / f'{field.name}.log').read_text().strip() for field in dataclasses.fields(cls)}
+        self = cls(**data)
+        return self
+
+    @property
+    def env(self):
+        return env.get_dict()
+
+
+
 def trace(is_debug=None, host=None, port=None, stdoutToServer=True, stderrToServer=True, **kwargs):
     """
 
@@ -28,4 +62,29 @@ def trace(is_debug=None, host=None, port=None, stdoutToServer=True, stderrToServ
     msg = MASK.format(host=host, port=port)
     logger.info(msg)
 
+    import pydevd_pycharm
     pydevd_pycharm.settrace(host, port=port, stdoutToServer=stdoutToServer, stderrToServer=stderrToServer, **kwargs)
+
+
+def debug_shell():
+    """
+
+    Starts a debug shell by initializing a `ShellDebug` object from a given path
+    and enabling tracing with debug mode turned on.
+
+    """
+    import sys
+    path_str = sys.argv[1]
+    data = ShellDebug.from_path(path_str)
+    trace(is_debug=True)
+    data
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv = [
+        'test.py',
+        './fmtr-debug/34e8d492-2f15-419a-8fcb-fe4fa0fa02bb',
+    ]
+    debug_shell()

fmtr/tools/dns_tools/__init__.py

@@ -0,0 +1,7 @@
+from fmtr.tools.import_tools import MissingExtraMockModule
+
+try:
+    from fmtr.tools.dns_tools import server, client, dm, proxy
+    import dns
+except ModuleNotFoundError as exception:
+    dns = server = client = dm = proxy = MissingExtraMockModule('dns', exception)