fmtr.tools 1.1.1__py3-none-any.whl → 1.3.81__py3-none-any.whl

fmtr/tools/logging_tools.py

@@ -4,23 +4,34 @@ import os
 from fmtr.tools import environment_tools
 from fmtr.tools.constants import Constants
 
-DEVELOPMENT = "development"
-PRODUCTION = "production"
-STREAM_DEFAULT = DEVELOPMENT
-ENVIRONMENT_DEFAULT = DEVELOPMENT
+if environment_tools.IS_DEV:
+    STREAM_DEFAULT = ENVIRONMENT_DEFAULT = Constants.DEVELOPMENT
+else:
+    STREAM_DEFAULT = None
+    ENVIRONMENT_DEFAULT = Constants.PRODUCTION
 
-LEVEL_DEFAULT = logging.DEBUG if environment_tools.IS_DEBUG else logging.INFO
+IS_DEBUG = environment_tools.get(Constants.FMTR_LOG_LEVEL_KEY, None, converter=str.upper) == 'DEBUG'
+LEVEL_DEFAULT = logging.DEBUG if IS_DEBUG else logging.INFO
 
 
+def null_scrubber(match):
+    """
+
+    Effectively disable scrubbing
+
+    """
+    return match.value
+
 def get_logger(name, version=None, host=Constants.FMTR_OBS_HOST, key=None, org=Constants.ORG_NAME,
-               stream=STREAM_DEFAULT,
-               environment=ENVIRONMENT_DEFAULT, level=LEVEL_DEFAULT):
+               stream=STREAM_DEFAULT, environment=ENVIRONMENT_DEFAULT, level=LEVEL_DEFAULT):
     """
 
     Get a pre-configured logfire logger, if dependency is present, otherwise default to native logger.
 
     """
 
+    stream = stream or name
+
     try:
         import logfire
     except ImportError:
@@ -30,30 +41,45 @@ def get_logger(name, version=None, host=Constants.FMTR_OBS_HOST, key=None, org=C
 
         return logger
 
+    logger = logfire
+
     if key is None:
-        key = environment_tools.get(Constants.FMTR_OBS_API_KEY_KEY)
-    url = f"https://{host}/api/{org}/v1/traces"
-    headers = f"Authorization=Basic {key},stream-name={stream}"
+        key = environment_tools.get(Constants.FMTR_OBS_API_KEY_KEY, default=None)
+
+    if key:
+        url = f"https://{host}/api/{org}/v1/traces"
+        headers = f"Authorization=Basic {key},stream-name={stream}"
 
-    os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = url
-    os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers
-    os.environ["OTEL_EXPORTER_OTLP_INSECURE"] = str(False).lower()
+        os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = url
+        os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers
+        os.environ["OTEL_EXPORTER_OTLP_INSECURE"] = str(False).lower()
 
     if not version:
         from fmtr.tools import version_tools
         version = version_tools.read()
 
+    # Rigmarole to translate native levels to logfire/otel ones.
+    lev_num_otel = logfire._internal.constants.LOGGING_TO_OTEL_LEVEL_NUMBERS[level]
+    lev_name_otel = logfire._internal.constants.NUMBER_TO_LEVEL[lev_num_otel]
+
+    console_opts = logfire.ConsoleOptions(
+        colors='always',
+        min_log_level=lev_name_otel,
+    )
+
     logfire.configure(
         service_name=name,
         service_version=version,
         environment=environment,
         send_to_logfire=False,
-        console=logfire.ConsoleOptions(colors='always' if environment_tools.IS_DEBUG else 'auto')
+        console=console_opts,
+        scrubbing=logfire.ScrubbingOptions(callback=null_scrubber)
     )
 
-    logging.getLogger(name).setLevel(level)
+    if key is None:
+        msg = f'Observability dependencies installed, but "{Constants.FMTR_OBS_API_KEY_KEY}" not set. Cloud observability will be disabled.'
+        logger.warning(msg)
 
-    logger = logfire
     return logger
 
 
@@ -63,3 +89,4 @@ if __name__ == '__main__':
     logger.info('Hello World')
     logger.warning('test warning')
     logger.debug('Hello World')
+    logger

fmtr/tools/packaging_tools.py

@@ -0,0 +1,14 @@
+import importlib.metadata
+
+
+def get_version(module):
+    """
+
+    Retrieve the version of a specified module.
+
+    """
+
+    if type(module) is not str:
+        module = module.__name__
+    version = importlib.metadata.version(module)
+    return version

fmtr/tools/path_tools/__init__.py

@@ -0,0 +1,12 @@
+from fmtr.tools.import_tools import MissingExtraMockModule
+from fmtr.tools.path_tools.path_tools import Path, PackagePaths
+
+try:
+    from fmtr.tools.path_tools.app_path_tools import AppPaths
+except ModuleNotFoundError as exception:
+    AppPaths = MissingExtraMockModule('path.app', exception)
+
+try:
+    from fmtr.tools.path_tools.type_path_tools import guess
+except ModuleNotFoundError as exception:
+    guess = MissingExtraMockModule('path.type', exception)

fmtr/tools/path_tools/app_path_tools.py

@@ -0,0 +1,40 @@
+import appdirs
+
+from fmtr.tools.path_tools import Path
+
+
+class AppPaths:
+    """
+
+    Wrap appdirs to return Path objects
+
+    """
+    PathType = Path
+
+    def user_data_dir(self, appname=None, appauthor=None, version=None, roaming=False):
+        path_str = appdirs.user_data_dir(appname=appname, appauthor=appauthor, version=version, roaming=roaming)
+        return self.PathType(path_str)
+
+    def user_config_dir(self, appname=None, appauthor=None, version=None, roaming=False):
+        path_str = appdirs.user_config_dir(appname=appname, appauthor=appauthor, version=version, roaming=roaming)
+        return self.PathType(path_str)
+
+    def site_config_dir(self, appname=None, appauthor=None, version=None):
+        path_str = appdirs.site_config_dir(appname=appname, appauthor=appauthor, version=version, multipath=False)
+        return self.PathType(path_str)
+
+    def site_data_dir(self, appname=None, appauthor=None, version=None):
+        path_str = appdirs.site_data_dir(appname=appname, appauthor=appauthor, version=version, multipath=False)
+        return self.PathType(path_str)
+
+    def user_cache_dir(self, appname=None, appauthor=None, version=None):
+        path_str = appdirs.user_cache_dir(appname=appname, appauthor=appauthor, version=version)
+        return self.PathType(path_str)
+
+    def user_state_dir(self, appname=None, appauthor=None, version=None):
+        path_str = appdirs.user_state_dir(appname=appname, appauthor=appauthor, version=version)
+        return self.PathType(path_str)
+
+    def user_log_dir(self, appname=None, appauthor=None, version=None):
+        path_str = appdirs.user_log_dir(appname=appname, appauthor=appauthor, version=version)
+        return self.PathType(path_str)

fmtr/tools/{path_tools.py → path_tools/path_tools.py}

@@ -145,19 +145,94 @@ class Path(type(Path())):
         """
         return self.mkdir(parents=True, exist_ok=True)
 
+    def with_suffix(self, suffix: str) -> 'Path':
+        """
+
+        Pathlib doesn't add a dot prefix, but then errors if you don't provide one, which feels rather obnoxious.
+
+        """
+        if not suffix.startswith('.'):
+            suffix = f'.{suffix}'
+        return super().with_suffix(suffix)
+
+    def get_conversion_path(self, suffix: str) -> 'Path':
+        """
+
+        Fetch the equivalent path for a different format in the standard conversion directory structure.
+        .../xyz/filename.xyx -> ../abc/filename.abc
+
+        """
+
+        old_dir = self.parent.name
+
+        if old_dir != self.suffix.removeprefix('.'):
+            raise ValueError(f"Expected parent directory '{old_dir}' to match file extension '{suffix}'")
+
+        new = self.parent.parent / suffix / f'{self.stem}.{suffix}'
+        return new
+
+    @property
+    def exist(self):
+        """
+
+        Exists as property
+
+        """
+        return super().exists()
+
+    @classmethod
+    def app(cls):
+        """
+
+        Convenience method for getting application paths
+
+        """
+        from fmtr.tools import path
+        return path.AppPaths()
+
+    @property
+    def type(self):
+        """
+
+        Infer file type, extension, etc.
+
+        """
+        if not self.exists():
+            return None
+        from fmtr.tools import path
+        kind = path.guess(str(self.absolute()))
+        return kind
+
+    @property
+    def children(self):
+        if not self.is_dir():
+            return None
+        return sorted(self.iterdir(), key=lambda x: x.is_dir(), reverse=True)
+
+
+class FromCallerMixin:
+    """
+
 
-class PackagePaths:
+    """
+
+    def from_caller(self):
+        from fmtr.tools.inspection_tools import get_call_path
+        path = get_call_path(offset=3).parent
+        return path
+
+
+class PackagePaths(FromCallerMixin):
     """
 
     Canonical paths for a package.
 
     """
 
-    FILENAME_CONFIG = 'settings.yaml'
-    DIR_NAME_ARTIFACTS = 'artifacts'
-    FILENAME_VERSION = 'version'
+    dev = Path('/') / 'opt' / 'dev'
+    data_global = dev / Constants.DIR_NAME_DATA
 
-    def __init__(self, path=None, org_singleton=None, dir_name_artifacts=DIR_NAME_ARTIFACTS, filename_config=FILENAME_CONFIG, file_version=FILENAME_VERSION):
+    def __init__(self, path=None, org_singleton=None, dir_name_data=Constants.DIR_NAME_DATA, filename_config=Constants.FILENAME_CONFIG, file_version=Constants.FILENAME_VERSION):
 
         """
 
@@ -165,15 +240,23 @@ class PackagePaths:
 
         """
         if not path:
-            from fmtr.tools.inspection_tools import get_call_path
-            path = get_call_path(offset=2).parent
+            path = self.from_caller()
 
         self.path = Path(path)
         self.org_singleton = org_singleton
-        self.dir_name_artifacts = dir_name_artifacts
+        self.dir_name_data = dir_name_data
         self.filename_config = filename_config
         self.filename_version = file_version
 
+    @property
+    def is_dev(self) -> bool:
+        """
+
+        Is the package in the dev directory - as opposed to `site-packages` etc?
+
+        """
+        return self.path.is_relative_to(self.dev)
+
     @property
     def is_namespace(self) -> bool:
         """
@@ -192,6 +275,19 @@ class PackagePaths:
         """
         return self.path.stem
 
+    @property
+    def name_ns(self) -> str:
+        """
+
+        Name of namespace package.
+
+        """
+
+        if self.is_namespace:
+            return f'{self.org}.{self.name}'
+        else:
+            return self.name
+
     @property
     def org(self) -> str:
         """
@@ -226,13 +322,46 @@ class PackagePaths:
         return self.path / self.filename_version
 
     @property
-    def artifacts(self) -> Path:
+    def data(self) -> Path:
+        """
+
+        Path of project-specific data directory.
+
+        """
+
+        return self.dev / Constants.DIR_NAME_REPO / self.name_ns / self.dir_name_data
+
+    @property
+    def cache(self) -> Path:
+        """
+
+        Path of cache directory.
+
+        """
+
+        return self.data / Constants.DIR_NAME_CACHE
+
+    @property
+    def artifact(self) -> Path:
+        """
+
+        Path of project-specific artifact directory
+
+        """
+
+        return self.data / Constants.DIR_NAME_ARTIFACT
+
+    @property
+    def source(self) -> Path:
         """
 
-        Path of artifacts directory.
+        Path of project-specific source directory
 
         """
-        return self.repo / self.dir_name_artifacts
+
+        return self.data / Constants.DIR_NAME_SOURCE
+
+
 
     @property
     def settings(self) -> Path:
@@ -241,7 +370,16 @@ class PackagePaths:
         Path of settings file.
 
         """
-        return self.artifacts / self.filename_config
+        return self.data / self.filename_config
+
+    @property
+    def hf(self) -> Path:
+        """
+
+        Path of HuggingFace directory
+
+        """
+        return self.artifact / Constants.DIR_NAME_HF
 
     def __repr__(self) -> str:
         """
@@ -250,3 +388,9 @@ class PackagePaths:
 
         """
         return f'{self.__class__.__name__}("{self.path}")'
+
+
+if __name__ == "__main__":
+    path = Path('/usr/bin/bash').absolute()
+    path.type
+    path

fmtr/tools/path_tools/type_path_tools.py

@@ -0,0 +1,3 @@
+import filetype
+
+guess = filetype.guess

fmtr/tools/pattern_tools.py

@@ -0,0 +1,260 @@
+import regex as re
+from dataclasses import dataclass, asdict
+from functools import cached_property
+from typing import List, Any
+
+from fmtr.tools import Constants
+from fmtr.tools.logging_tools import logger
+from fmtr.tools.string_tools import join
+
+
+class RewriteCircularLoopError(Exception):
+    """
+
+    Circular loop error
+
+    """
+
+
+MASK_GROUP = '(?:{pattern})'
+MASK_NAMED = r"(?P<{key}>{pattern})"
+
+
+def alt(*patterns):
+    patterns = sorted(patterns, key=len, reverse=True)
+    pattern = '|'.join(patterns)
+    pattern = MASK_GROUP.format(pattern=pattern)
+    return pattern
+
+@dataclass
+class Key:
+    RECORD_SEP = '␞'
+    FILLS = None
+
+    def flatten(self, data):
+        """
+
+        Flatten/serialise dictionary data
+
+        """
+        pairs = [f'{value}' for key, value in data.items()]
+        string = self.RECORD_SEP.join(pairs)
+        return string
+
+    @cached_property
+    def pattern(self):
+        """
+
+        Serialise to pattern
+
+        """
+        data = {
+            key:
+                MASK_NAMED.format(
+                    key=key,
+                    pattern=value.format_map(self.fills))
+            for key, value in asdict(self).items()
+        }
+        pattern = self.flatten(data)
+        return pattern
+
+    @cached_property
+    def rx(self):
+        """
+
+        Compile to Regular Expression
+
+        """
+        return re.compile(self.pattern)
+
+    @cached_property
+    def string(self):
+        """
+
+        Serialise to string
+
+        """
+        string = self.flatten(asdict(self))
+        return string
+
+    @cached_property
+    def fills(self):
+        """
+
+        Add key names as regex group names
+
+        """
+        return {key: MASK_NAMED.format(key=key, pattern=value) for key, value in self.FILLS.items()}
+
+
+    def transform(self, match: re.Match):
+        """
+
+        Transform match object into a new object of the same type.
+
+        """
+        fills = match.groupdict()
+        data = {key: value.format_map(fills) for key, value in asdict(self).items()}
+        obj = self.__class__(**data)
+        return obj
+
+
+@dataclass
+class Item:
+    """
+
+    Key-value pair
+
+    """
+    source: Key
+    target: Key
+
+
+@dataclass(kw_only=True)
+class Transformer:
+    """
+    
+    Pattern-based, dictionary-like mapper.
+    Compiles an complex set of rules into single regex pattern, and determines which rule matched.
+    Inputs are then transformed according to the matching rule.
+    Works like a pattern-based dictionary when is_recursive==False.
+    Works something like an FSA/transducer when is_recursive=True.
+
+    """
+    PREFIX_GROUP = '__'
+    items: List[Item]
+    default: Any = None
+    is_recursive: bool = False
+
+    def __post_init__(self):
+        with logger.span(f'Compiling expression {len(self.items)=}'):
+            rx = self.rx
+        logger.debug(f'Compiled successfully {rx.groups=}')
+
+    @cached_property
+    def pattern(self) -> str:
+        """
+        
+        Dynamically generated regex pattern based on the rules provided.
+
+        """
+        patterns = [
+            MASK_NAMED.format(key=f'{self.PREFIX_GROUP}{i}', pattern=item.source.pattern)
+            for i, item in enumerate(self.items)
+        ]
+        pattern = alt(*patterns)
+        return pattern
+
+    @cached_property
+    def rx(self) -> re.Pattern:
+        """
+
+        Regex object.
+
+        """
+        return re.compile(self.pattern)
+
+    def get_default(self, key: Key) -> Any:
+        """
+
+        Define what to return in case of no match
+
+        """
+        if self.is_recursive:
+            return key
+        else:
+            return self.default
+
+    def get(self, key: Key) -> Key | Any:
+        """
+
+        Use recursive or single lookup pass, depending on whether recursive lookups have been specified.
+
+        """
+        if self.is_recursive:
+            with logger.span(f'Transforming recursively {key=}...'):
+                return self.get_recursive(key)
+        else:
+            with logger.span(f'Transforming linearly {key=}...'):
+                return self.get_one(key)
+
+    def get_one(self, key: Key) -> Key | Any:
+        """
+
+        Single lookup pass.
+        Lookup the source string based on the matching rule.
+
+        """
+
+        match = self.rx.fullmatch(key.string)
+
+        if not match:
+            value = self.get_default(key)
+            logger.debug(f'No match for {key=}. Returning {self.get_default(key)=}')
+        else:
+
+            match_ids = {name: v for name, v in match.groupdict().items() if v}
+            rule_ids = {
+                int(id.removeprefix(self.PREFIX_GROUP))
+                for id in match_ids.keys() if id.startswith(self.PREFIX_GROUP)
+            }
+
+            if len(rule_ids) != 1:
+                msg = f'Multiple group matches: {rule_ids}'
+                raise ValueError(msg)
+
+            rule_id = next(iter(rule_ids))
+            rule = self.items[rule_id]
+
+            logger.debug(f'Matched using {rule_id=}: {rule.source=}')
+
+            if isinstance(rule.target, Key):
+                value = rule.target.transform(match)
+            else:
+                value = rule.target
+
+            logger.debug(f'Transformed using {rule_id=}: {key=} → {value=}')
+
+        return value
+
+    def get_recursive(self, key: Key) -> Key | Any:
+        """
+
+        Lookup the provided key by continuously applying transforms until no changes are made
+        or a circular loop is detected.
+
+        """
+        history = []
+        previous = key
+
+        def get_history_str():
+            return join(history, sep=Constants.ARROW_SEP)
+
+        while True:
+            if previous in history:
+                history.append(previous)
+                msg = f'Loop detected on node "{previous}": {get_history_str()}'
+                raise RewriteCircularLoopError(msg)
+
+            history.append(previous)
+            new = previous
+            new = self.get_one(new)
+            if new == previous:
+                break
+            previous = new
+
+            if not isinstance(new, Key):
+                history.append(previous)
+                break
+
+        if len(history) == 1:
+            history_str = 'No transforms performed.'
+        else:
+            history_str = get_history_str()
+        logger.debug(f'Finished transforming: {history_str}')
+
+        return previous
+
+
+if __name__ == '__main__':
+    ...