fmtr.tools 1.1.1__py3-none-any.whl → 1.3.81__py3-none-any.whl

fmtr/tools/constants.py

@@ -14,6 +14,10 @@ class Constants:
     DATETIME_NOW_STR = DATETIME_NOW.strftime(DATETIME_FILENAME_FORMAT)
     SERIALIZATION_INDENT = 4
 
+    ARROW = '→'
+    ARROW_SEP = f' {ARROW} '
+
+    FMTR_DEV_KEY = 'FMTR_DEV'
     FMTR_LOG_LEVEL_KEY = 'FMTR_LOG_LEVEL'
     FMTR_OBS_API_KEY_KEY = 'FMTR_OBS_API_KEY'
     FMTR_OBS_HOST = 'obs.sv.fmtr.dev'
@@ -27,3 +31,32 @@ class Constants:
 
     FMTR_AI_HOST_KEY = 'FMTR_URL_HOST'
     FMTR_AI_HOST_DEFAULT = 'ai.gex.fmtr.dev'
+
+    FMTR_DEV_HOST = 'ws.gex.fmtr.dev'
+
+    FMTR_DEV_INTERFACE_URL = f'https://{FMTR_DEV_HOST}/'
+    FMTR_DEV_INTERFACE_SUB_URL_MASK = f'https://{{sub}}.{FMTR_DEV_HOST}/'
+
+    FILENAME_CONFIG = 'settings.yaml'
+    DIR_NAME_REPO = 'repo'
+    DIR_NAME_DATA = 'data'
+    DIR_NAME_CACHE = 'cache'
+    DIR_NAME_ARTIFACT = 'artifact'
+    DIR_NAME_SOURCE = 'source'
+    FILENAME_VERSION = 'version'
+    DIR_NAME_HF = 'hf'
+
+    ENTRYPOINT = 'entrypoint'
+    ENTRYPOINTS_DIR = f'{ENTRYPOINT}s'
+    ENTRYPOINT_FILE = f'{ENTRYPOINT}.py'
+
+    PACKAGE_EXCLUDE_DIRS = {'data', 'build', 'dist', '.*', '*egg-info*'}
+    INIT_FILENAME = '__init__.py'
+
+    DEVELOPMENT = "development"
+    PRODUCTION = "production"
+
+    INFRA = 'infra'
+
+    PROMPT_NONE_SPECIFIED = '[None Specified]'
+    WEBHOOK_URL_NOTIFY_KEY = 'WEBHOOK_URL_NOTIFY'

fmtr/tools/context_tools.py

@@ -0,0 +1,23 @@
+from contextlib import contextmanager, ExitStack
+
+
+@contextmanager
+def null():
+    """
+
+    Null context manager.
+
+    """
+    yield
+
+
+@contextmanager
+def contexts(*contexts):
+    """
+
+    Tee context managers.
+
+    """
+    with ExitStack() as stack:
+        resources = [stack.enter_context(context) for context in contexts]
+        yield resources

fmtr/tools/data_modelling_tools.py

@@ -1,4 +1,135 @@
-from pydantic import BaseModel, RootModel, ConfigDict
+import inspect
+from functools import cached_property
+from typing import ClassVar, List, Any, Dict
+
+from pydantic import BaseModel
+from pydantic import RootModel, ConfigDict
+from pydantic.fields import FieldInfo
+from pydantic.json_schema import SkipJsonSchema
+from pydantic_core import PydanticUndefined, PydanticUndefinedType
+
+from fmtr.tools.datatype_tools import is_optional, none_else
+from fmtr.tools.iterator_tools import get_class_lookup
+from fmtr.tools.string_tools import camel_to_snake
+from fmtr.tools.tools import Auto, Required, Empty
+
+
+class Field(FieldInfo):
+    """
+
+    Allow DRYer field definitions, set annotation and defaults at the same time, easier field inheritance, etc.
+
+    """
+    NAME = Auto
+    ANNOTATION = Empty
+    DEFAULT = Auto
+    FILLS = None
+    DESCRIPTION = None
+    TITLE = Auto
+    SKIP_SCHEMA = False
+    CONFIG = None
+
+    def __init__(self, annotation=Empty, default=Empty, description=None, title=None, fills=None, skip_schema=None, **kwargs):
+        """
+
+        Infer default from type annotation, if enabled, use class/argument fills to create titles/descriptions, etc.
+
+        """
+
+        fills_super = getattr(super(), 'FILLS', None)
+        self.fills = (fills_super or {}) | (self.FILLS or {}) | (fills or {})
+
+        skip_schema = none_else(skip_schema, self.SKIP_SCHEMA)
+
+        self.annotation = self.ANNOTATION if annotation is Empty else annotation
+        if self.annotation is Empty:
+            raise ValueError("Annotation must be specified.")
+
+        if skip_schema:
+            self.annotation = SkipJsonSchema[self.annotation]
+
+        default = self.get_default_auto(default)
+        if default is Required:
+            default = PydanticUndefined
+
+        description = self.get_desc(description)
+        title = self.get_title_auto(title)
+        kwargs |= (self.CONFIG or {})
+
+        super().__init__(annotation=self.annotation, default=default, title=title, description=description, **kwargs)
+
+    @classmethod
+    def get_name_auto(cls) -> str:
+        """
+
+        Infer field name, if set to auto.
+
+        """
+        if cls.NAME is Auto:
+            return camel_to_snake(cls.__name__)
+        elif cls.NAME is None:
+            return cls.__name__
+
+        return cls.NAME
+
+    @cached_property
+    def fills(self) -> Dict[str, str]:
+        """
+
+        Get fills with filled title merged in
+
+        """
+
+        fills_super = getattr(super(), 'FILLS', None)
+
+        return (fills_super or {}) | (self.FILLS or {}) | dict(title=self.get_title_auto())
+
+    def get_default_auto(self, default) -> type[Any] | None | PydanticUndefinedType:
+        """
+
+        Infer default, if not specified.
+
+        """
+
+        if default is not Empty:
+            return default
+
+        if self.DEFAULT is not Auto:
+            return self.DEFAULT
+
+        if is_optional(self.annotation):
+            return None
+        else:
+            return Required
+
+    def get_title_auto(self, mask) -> str | None:
+        """
+
+        Get title from classname/mask
+
+        """
+
+        if not mask:
+            mask = self.__class__.__name__ if self.TITLE is Auto else self.TITLE
+
+        if mask:
+            return mask.format(**self.fills)
+
+        return None
+
+    def get_desc(self, mask) -> str | None:
+        """
+
+        Fill description mask, if specified
+
+        """
+
+        mask = mask or self.DESCRIPTION
+
+        if mask:
+            return mask.format(**self.fills)
+
+        return None
 
 
 def to_df(*objs, name_value='value'):
@@ -51,30 +182,102 @@ class MixinFromJson:
         return self
 
 
-class Base(BaseModel, MixinFromJson):
+class CliRunMixin:
+    """
+
+    Mixin only so that it can also be used with Pydantic Settings.
+
+    TODO Ideally, the run method would be defined on dm.Base and the settings base would just inherit like set.Base(BaseSettings, dm.Base), but this isn't yet tested/could break Fields.
+
+    """
+
+    def run(self):
+        """
+
+        Behaviour when run as a CLI command, i.e. via Pydantic Settings. Run any subcommands then exit.
+
+        """
+
+        from pydantic_settings import get_subcommand
+
+        command = get_subcommand(self, is_required=False, cli_exit_on_error=False)
+
+        if not command:
+            return
+
+        result = command.run()
+        if inspect.isawaitable(result):
+            import asyncio
+            result = asyncio.run(result)
+
+        raise SystemExit(result)
+
+
+class Base(BaseModel, MixinFromJson, CliRunMixin):
     """
 
-    Base model
+    Base model allowing model definition via a list of custom Field objects.
 
     """
+    FIELDS: ClassVar[List[Field] | Dict[str, Field]] = []
 
-    def to_df(self, name_value='value'):
+    def __init_subclass__(cls, **kwargs):
         """
 
-        DataFrame representation with Fields as rows.
+        Fetch aggregated fields metadata from the hierarchy and set annotations and FieldInfo objects in the class.
 
         """
+        super().__init_subclass__(**kwargs)
 
-        objs = []
-        for name in self.model_fields.keys():
-            val = getattr(self, name)
-            objs.append(val)
+        fields = {}
+        for base in reversed(cls.__mro__):
 
-        df = to_df(*objs, name_value=name_value)
-        df['id'] = list(self.model_fields.keys())
-        df = df.set_index('id', drop=True)
+            try:
+                raw = base.FIELDS
+            except AttributeError:
+                raw = {}
+
+            if isinstance(raw, dict):
+                fields |= raw
+            else:
+                fields |= get_class_lookup(*raw, name_function=lambda cls_field: cls_field.get_name_auto())
+
+        cls.FIELDS = fields
+
+        for name, field in fields.items():
+            if name in cls.__annotations__:
+                continue
+
+            if inspect.isclass(field):
+                field = field()
+
+            setattr(cls, name, field)
+            cls.__annotations__[name] = field.annotation
+
+    def to_df(self, **kwargs):
+        """
+
+        DataFrame representation of Data Model.
+
+        """
+        from fmtr.tools import tabular
+
+        data = self.model_dump(**kwargs)
+        df = tabular.pd.json_normalize(data, sep='_')
         return df
 
+    @classmethod
+    def to_df_empty(cls):
+        """
+
+        Empty DataFrame
+
+        """
+        from fmtr.tools import tabular
+
+        row = {name: None for name in cls.model_fields.keys()}
+        df = tabular.pd.DataFrame([row])
+        return df
 
 class Root(RootModel, MixinFromJson):
     """
@@ -83,11 +286,21 @@ class Root(RootModel, MixinFromJson):
 
     """
 
-    def to_df(self):
+    def to_df(self, **kwargs):
         """
 
         DataFrame representation with items as rows.
 
         """
 
-        return to_df(*self.items)
+        """
+
+        DataFrame representation of Data Model.
+
+        """
+        from fmtr.tools import tabular
+
+        data = [item.model_dump(**kwargs) for item in self.items]
+        dfs = [tabular.pd.json_normalize(datum, sep='_') for datum in data]
+        df = tabular.pd.concat(dfs, axis=tabular.CONCAT_VERTICALLY)
+        return df

fmtr/tools/database_tools/__init__.py

@@ -0,0 +1,6 @@
+from fmtr.tools.import_tools import MissingExtraMockModule
+
+try:
+    from fmtr.tools.database_tools import document
+except ModuleNotFoundError as exception:
+    document = MissingExtraMockModule('db.document', exception)

fmtr/tools/database_tools/document.py

@@ -0,0 +1,51 @@
+from functools import cached_property
+from typing import List
+
+import beanie
+from beanie.odm import actions
+from pymongo import AsyncMongoClient
+
+from fmtr.tools import data_modelling_tools
+from fmtr.tools.constants import Constants
+from fmtr.tools.logging_tools import logger
+
+ModifyEvents = [
+    actions.Insert,
+    actions.Replace,
+    actions.Save,
+    actions.SaveChanges,
+    actions.Update
+]
+
+
+class Document(beanie.Document, data_modelling_tools.Base):
+    """
+
+    Document stub.
+
+    """
+
+
+class Client:
+
+    def __init__(self, name, host=Constants.FMTR_DEV_HOST, port=27017, documents: List[beanie.Document] | None = None):
+        self.name = name
+        self.host = host
+        self.port = port
+        self.documents = documents
+
+        self.client = AsyncMongoClient(self.uri, tz_aware=True)
+        self.db = self.client[self.name]
+
+    @cached_property
+    def uri(self):
+        return f'mongodb://{self.host}:{self.port}'
+
+    async def connect(self):
+        """
+
+        Connect
+
+        """
+        with logger.span(f'Connecting to document database {self.uri=} {self.name=}'):
+            return await beanie.init_beanie(database=self.db, document_models=self.documents)

fmtr/tools/datatype_tools.py

@@ -1,4 +1,5 @@
-from typing import Any
+from types import UnionType, NoneType
+from typing import Any, get_origin, get_args, Union, Annotated
 
 from fmtr.tools.tools import Raise
 
@@ -84,3 +85,22 @@ def none_else(value: Any, default: Any) -> Any:
     if is_none(value):
         return default
     return value
+
+
+def is_optional(annotation) -> bool:
+    """
+
+    Is type/annotation optional? todo should be in typing_tools?
+
+    """
+    if annotation is None:
+        return True
+
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+
+    if origin is Annotated:
+        return is_optional(args[0])
+
+    is_opt = (origin is UnionType or origin is Union) and NoneType in args
+    return is_opt

fmtr/tools/datetime_tools.py

@@ -0,0 +1,12 @@
+from datetime import datetime, timezone
+
+MIN = datetime.min.replace(tzinfo=timezone.utc)
+MAX = datetime.max.replace(tzinfo=timezone.utc)
+
+def now() -> datetime:
+    """
+
+    Now UTC
+
+    """
+    return datetime.now(tz=timezone.utc)

fmtr/tools/debugging_tools.py

@@ -1,3 +1,5 @@
+import dataclasses
+
 import pydevd_pycharm
 
 from fmtr.tools import environment_tools as env
@@ -5,6 +7,40 @@ from fmtr.tools.constants import Constants
 
 MASK = 'Starting debugger at tcp://{host}:{port}...'
 
+
+@dataclasses.dataclass
+class ShellDebug:
+    """
+
+    Debugging information for shell commands
+
+    """
+    command: str
+    out: str
+    err: str
+    status: int
+
+    # timestamp: datetime=dataclasses.field(default_factory=datetime.now)
+
+    @classmethod
+    def from_path(cls, path_str):
+        """
+
+        Get debug info from path
+
+        """
+        from fmtr.tools import Path
+        path = Path(path_str).absolute()
+        data = {field.name: (path / f'{field.name}.log').read_text().strip() for field in dataclasses.fields(cls)}
+        self = cls(**data)
+        return self
+
+    @property
+    def env(self):
+        return env.get_dict()
+
+
+
 def trace(is_debug=None, host=None, port=None, stdoutToServer=True, stderrToServer=True, **kwargs):
     """
 
@@ -29,3 +65,27 @@ def trace(is_debug=None, host=None, port=None, stdoutToServer=True, stderrToServ
     logger.info(msg)
 
     pydevd_pycharm.settrace(host, port=port, stdoutToServer=stdoutToServer, stderrToServer=stderrToServer, **kwargs)
+
+
+def debug_shell():
+    """
+
+    Starts a debug shell by initializing a `ShellDebug` object from a given path
+    and enabling tracing with debug mode turned on.
+
+    """
+    import sys
+    path_str = sys.argv[1]
+    data = ShellDebug.from_path(path_str)
+    trace(is_debug=True)
+    data
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.argv = [
+        'test.py',
+        './fmtr-debug/34e8d492-2f15-419a-8fcb-fe4fa0fa02bb',
+    ]
+    debug_shell()

fmtr/tools/dns_tools/__init__.py

@@ -0,0 +1,7 @@
+from fmtr.tools.import_tools import MissingExtraMockModule
+
+try:
+    from fmtr.tools.dns_tools import server, client, dm, proxy
+    import dns
+except ModuleNotFoundError as exception:
+    dns = server = client = dm = proxy = MissingExtraMockModule('dns', exception)

fmtr/tools/dns_tools/client.py

@@ -0,0 +1,97 @@
+from dataclasses import dataclass
+from dns import query as dnspython_query, message as dnspython_message, rdatatype as dnspython_rdatatype, rcode as dnspython_rcode
+from functools import cached_property
+from httpx_retries import Retry, RetryTransport
+from typing import Optional
+
+from fmtr.tools import http_tools as http
+from fmtr.tools.dns_tools.dm import Exchange, Response
+from fmtr.tools.logging_tools import logger
+
+RETRY_STRATEGY = Retry(
+    total=2,  # initial + 1 retry
+    allowed_methods={"GET", "POST"},
+    status_forcelist={502, 503, 504},
+    retry_on_exceptions=None,  # defaults to httpx.TransportError etc.
+    backoff_factor=0.25,  # short backoff (e.g. 0.25s, 0.5s)
+    max_backoff_wait=0.75,  # max total delay before giving up
+    backoff_jitter=0.1,  # small jitter to avoid retry bursts
+    respect_retry_after_header=False,  # DoH resolvers probably won't set this
+)
+
+
+class HTTPClientDoH(http.Client):
+    """
+
+    Base HTTP client for DoH-appropriate retry strategy.
+
+    """
+    TRANSPORT = RetryTransport(retry=RETRY_STRATEGY)
+
+
+@dataclass
+class Plain:
+    """
+
+    Plain DNS
+
+    """
+    host: str
+    port: int = 53
+    ttl_min: Optional[int] = None
+
+    def resolve(self, exchange: Exchange):
+
+        with logger.span(f'UDP {self.host}:{self.port}'):
+            response_plain = dnspython_query.udp(q=exchange.query_last, where=self.host, port=self.port)
+            response = Response.from_message(response_plain)
+            for answer in response.message.answer:
+                answer.ttl = max(answer.ttl, self.ttl_min or answer.ttl)
+
+        exchange.response = response
+
+
+@dataclass
+class HTTP:
+    """
+
+    DNS over HTTP
+
+    """
+
+    HEADERS = {"Content-Type": "application/dns-message"}
+    CLIENT = HTTPClientDoH()
+    BOOTSTRAP = Plain('8.8.8.8')
+
+    host: str
+    url: str
+
+
+    @cached_property
+    def ip(self):
+        message = dnspython_message.make_query(self.host, dnspython_rdatatype.A, flags=0)
+        exchange = Exchange.from_wire(message.to_wire(), ip=None, port=None)
+        self.BOOTSTRAP.resolve(exchange)
+        ip = next(iter(exchange.response.answer.items.keys())).address
+        return ip
+
+    def resolve(self, exchange: Exchange):
+        """
+
+        Resolve via DoH
+
+        """
+
+        headers = self.HEADERS | dict(Host=self.host)
+        url = self.url.format(host=self.ip)
+
+        try:
+            response_doh = self.CLIENT.post(url, headers=headers, content=exchange.query_last.to_wire())
+            response_doh.raise_for_status()
+            response = Response.from_http(response_doh)
+            exchange.response = response
+
+        except Exception as exception:
+            exchange.response.message.set_rcode(dnspython_rcode.SERVFAIL)
+            exchange.response.is_complete = True
+            logger.exception(exception)