fmtr.tools 1.1.1__py3-none-any.whl → 1.4.37__py3-none-any.whl

fmtr/tools/{path_tools.py → path_tools/path_tools.py}

@@ -1,8 +1,9 @@
+from pathlib import Path
+
 import re
 import subprocess
-from pathlib import Path
 from tempfile import gettempdir
-from typing import Union, Any
+from typing import Union, Any, Self
 
 from fmtr.tools.constants import Constants
 from fmtr.tools.platform_tools import is_wsl
@@ -145,19 +146,134 @@ class Path(type(Path())):
         """
         return self.mkdir(parents=True, exist_ok=True)
 
+    def with_suffix(self, suffix: str) -> 'Path':
+        """
+
+        Pathlib doesn't add a dot prefix, but then errors if you don't provide one, which feels rather obnoxious.
+
+        """
+        if not suffix.startswith('.'):
+            suffix = f'.{suffix}'
+        return super().with_suffix(suffix)
+
+    def get_conversion_path(self, suffix: str) -> 'Path':
+        """
+
+        Fetch the equivalent path for a different format in the standard conversion directory structure.
+        .../xyz/filename.xyx -> ../abc/filename.abc
+
+        """
+
+        old_dir = self.parent.name
+
+        if old_dir != self.suffix.removeprefix('.'):
+            raise ValueError(f"Expected parent directory '{old_dir}' to match file extension '{suffix}'")
+
+        new = self.parent.parent / suffix / f'{self.stem}.{suffix}'
+        return new
+
+    @property
+    def exist(self):
+        """
+
+        Exists as property
+
+        """
+        return super().exists()
+
+    @classmethod
+    def app(cls):
+        """
+
+        Convenience method for getting application paths
+
+        """
+        from fmtr.tools import path
+        return path.AppPaths()
+
+    @property
+    def type(self):
+        """
+
+        Infer file type, extension, etc.
+
+        """
+        if not self.exists():
+            return None
+        from fmtr.tools import path
+        kind = path.guess(str(self.absolute()))
+        return kind
+
+    @property
+    def children(self) -> list[Self]:
+        """
+
+        Recursive children property
+
+        """
+        if not self.is_dir():
+            return None
+        return sorted(self.iterdir(), key=lambda x: x.is_dir(), reverse=True)
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source, handler):
+        """
+
+        Support Pydantic de/serialization and validation
+
+        TODO: Ideally these would be a mixin in dm, but then we'd need Pydantic to use it. Split dm module into Pydantic depts and other utils and import from there.
+
+        """
+        from pydantic_core import core_schema
+        return core_schema.no_info_plain_validator_function(
+            cls.__deserialize_pydantic__,
+            serialization=core_schema.plain_serializer_function_ser_schema(cls.__serialize_pydantic__),
+        )
+
+    @classmethod
+    def __serialize_pydantic__(cls, self) -> str:
+        """
+
+        Serialize to string
+
+        """
+        return str(self)
+
+    @classmethod
+    def __deserialize_pydantic__(cls, data) -> Self:
+        """
+
+        Deserialize from string
+
+        """
+        if isinstance(data, cls):
+            return data
+        return cls(data)
+
+
+class FromCallerMixin:
+    """
+
+
+    """
+
+    def from_caller(self):
+        from fmtr.tools.inspection_tools import get_call_path
+        path = get_call_path(offset=3).parent
+        return path
 
-class PackagePaths:
+
+class PackagePaths(FromCallerMixin):
     """
 
     Canonical paths for a package.
 
     """
 
-    FILENAME_CONFIG = 'settings.yaml'
-    DIR_NAME_ARTIFACTS = 'artifacts'
-    FILENAME_VERSION = 'version'
+    dev = Path('/') / 'opt' / 'dev'
+    data_global = dev / Constants.DIR_NAME_DATA
 
-    def __init__(self, path=None, org_singleton=None, dir_name_artifacts=DIR_NAME_ARTIFACTS, filename_config=FILENAME_CONFIG, file_version=FILENAME_VERSION):
+    def __init__(self, path=None, org_singleton=None, dir_name_data=Constants.DIR_NAME_DATA, filename_config=Constants.FILENAME_CONFIG, file_version=Constants.FILENAME_VERSION):
 
         """
 
@@ -165,15 +281,23 @@ class PackagePaths:
 
         """
         if not path:
-            from fmtr.tools.inspection_tools import get_call_path
-            path = get_call_path(offset=2).parent
+            path = self.from_caller()
 
         self.path = Path(path)
         self.org_singleton = org_singleton
-        self.dir_name_artifacts = dir_name_artifacts
+        self.dir_name_data = dir_name_data
         self.filename_config = filename_config
         self.filename_version = file_version
 
+    @property
+    def is_dev(self) -> bool:
+        """
+
+        Is the package in the dev directory - as opposed to `site-packages` etc?
+
+        """
+        return self.path.is_relative_to(self.dev)
+
     @property
     def is_namespace(self) -> bool:
         """
@@ -192,6 +316,19 @@ class PackagePaths:
         """
         return self.path.stem
 
+    @property
+    def name_ns(self) -> str:
+        """
+
+        Name of namespace package.
+
+        """
+
+        if self.is_namespace:
+            return f'{self.org}.{self.name}'
+        else:
+            return self.name
+
     @property
     def org(self) -> str:
         """
@@ -226,13 +363,44 @@ class PackagePaths:
         return self.path / self.filename_version
 
     @property
-    def artifacts(self) -> Path:
+    def data(self) -> Path:
+        """
+
+        Path of project-specific data directory.
+
+        """
+
+        return self.dev / Constants.DIR_NAME_REPO / self.name_ns / self.dir_name_data
+
+    @property
+    def cache(self) -> Path:
         """
 
-        Path of artifacts directory.
+        Path of cache directory.
 
         """
-        return self.repo / self.dir_name_artifacts
+
+        return self.data / Constants.DIR_NAME_CACHE
+
+    @property
+    def artifact(self) -> Path:
+        """
+
+        Path of project-specific artifact directory
+
+        """
+
+        return self.data / Constants.DIR_NAME_ARTIFACT
+
+    @property
+    def source(self) -> Path:
+        """
+
+        Path of project-specific source directory
+
+        """
+
+        return self.data / Constants.DIR_NAME_SOURCE
 
     @property
     def settings(self) -> Path:
@@ -241,7 +409,34 @@ class PackagePaths:
         Path of settings file.
 
         """
-        return self.artifacts / self.filename_config
+        return self.data / self.filename_config
+
+    @property
+    def hf(self) -> Path:
+        """
+
+        Path of HuggingFace directory
+
+        """
+        return self.artifact / Constants.DIR_NAME_HF
+
+    @property
+    def docs(self) -> Path:
+        """
+
+        Path of docs directory
+
+        """
+        return self.repo / Constants.DOCS_DIR
+
+    @property
+    def docs_config(self) -> Path:
+        """
+
+        Path of docs config file
+
+        """
+        return self.repo / Constants.DOCS_CONFIG_FILENAME
 
     def __repr__(self) -> str:
         """
@@ -250,3 +445,11 @@ class PackagePaths:
 
         """
         return f'{self.__class__.__name__}("{self.path}")'
+
+
+root = Path('/')
+
+if __name__ == "__main__":
+    path = Path('/usr/bin/bash').absolute()
+    path.type
+    path

fmtr/tools/path_tools/type_path_tools.py

@@ -0,0 +1,3 @@
+import filetype
+
+guess = filetype.guess

fmtr/tools/pattern_tools.py

@@ -0,0 +1,277 @@
+import regex as re
+from dataclasses import dataclass, asdict
+from functools import cached_property
+from typing import List, Any
+
+from fmtr.tools import Constants
+from fmtr.tools.logging_tools import logger
+from fmtr.tools.string_tools import join
+
+
+class RewriteCircularLoopError(Exception):
+    """
+
+    Circular loop error
+
+    """
+
+
+MASK_GROUP = '(?:{pattern})'
+MASK_NAMED = r"(?P<{key}>{pattern})"
+
+
+def alt(*patterns):
+    patterns = sorted(patterns, key=len, reverse=True)
+    pattern = '|'.join(patterns)
+    pattern = MASK_GROUP.format(pattern=pattern)
+    return pattern
+
+@dataclass
+class Key:
+    RECORD_SEP = '␞'
+    FILLS = None
+
+    def flatten(self, data):
+        """
+
+        Flatten/serialise dictionary data
+
+        """
+        pairs = [f'{value}' for key, value in data.items()]
+        string = self.RECORD_SEP.join(pairs)
+        return string
+
+    @cached_property
+    def pattern(self):
+        """
+
+        Serialise to pattern
+
+        """
+        data = {
+            key:
+                MASK_NAMED.format(
+                    key=key,
+                    pattern=value.format_map(self.fills))
+            for key, value in asdict(self).items()
+        }
+        pattern = self.flatten(data)
+        return pattern
+
+    @cached_property
+    def rx(self):
+        """
+
+        Compile to Regular Expression
+
+        """
+        return re.compile(self.pattern)
+
+    @cached_property
+    def string(self):
+        """
+
+        Serialise to string
+
+        """
+        string = self.flatten(asdict(self))
+        return string
+
+    @cached_property
+    def fills(self):
+        """
+
+        Add key names as regex group names
+
+        """
+        return {key: MASK_NAMED.format(key=key, pattern=value) for key, value in self.FILLS.items()}
+
+
+    def transform(self, match: re.Match):
+        """
+
+        Transform match object into a new object of the same type.
+
+        """
+        fills = match.groupdict()
+        data = {key: value.format_map(fills) for key, value in asdict(self).items()}
+        obj = self.__class__(**data)
+        return obj
+
+
+@dataclass
+class Item:
+    """
+
+    Key-value pair
+
+    """
+    source: Key
+    target: Key
+
+
+@dataclass(kw_only=True)
+class Transformer:
+    """
+    
+    Pattern-based, dictionary-like mapper.
+    Compiles an complex set of rules into single regex pattern, and determines which rule matched.
+    Inputs are then transformed according to the matching rule.
+    Works like a pattern-based dictionary when is_recursive==False.
+    Works something like an FSA/transducer when is_recursive=True.
+
+    """
+    PREFIX_GROUP = '__'
+    items: List[Item]
+    default: Any = None
+    is_recursive: bool = False
+
+    def __post_init__(self):
+        """
+
+        Compile on init
+
+        """
+        return self.compile(clear=False)
+
+    def compile(self, clear=True):
+        """
+
+        Re/compile regex pattern, invalidating existing caches if recompile.
+
+        """
+        if clear:
+            del self.pattern
+            del self.rx
+
+        with logger.span(f'Compiling expression {len(self.items)=}'):
+            rx = self.rx
+        logger.debug(f'Compiled successfully {rx.groups=}')
+
+    @cached_property
+    def pattern(self) -> str:
+        """
+        
+        Dynamically generated regex pattern based on the rules provided.
+
+        """
+        patterns = [
+            MASK_NAMED.format(key=f'{self.PREFIX_GROUP}{i}', pattern=item.source.pattern)
+            for i, item in enumerate(self.items)
+        ]
+        pattern = alt(*patterns)
+        return pattern
+
+    @cached_property
+    def rx(self) -> re.Pattern:
+        """
+
+        Regex object.
+
+        """
+        return re.compile(self.pattern)
+
+    def get_default(self, key: Key) -> Any:
+        """
+
+        Define what to return in case of no match
+
+        """
+        if self.is_recursive:
+            return key
+        else:
+            return self.default
+
+    def get(self, key: Key) -> Key | Any:
+        """
+
+        Use recursive or single lookup pass, depending on whether recursive lookups have been specified.
+
+        """
+        if self.is_recursive:
+            with logger.span(f'Transforming recursively {key=}...'):
+                return self.get_recursive(key)
+        else:
+            with logger.span(f'Transforming linearly {key=}...'):
+                return self.get_one(key)
+
+    def get_one(self, key: Key) -> Key | Any:
+        """
+
+        Single lookup pass.
+        Lookup the source string based on the matching rule.
+
+        """
+
+        match = self.rx.fullmatch(key.string)
+
+        if not match:
+            value = self.get_default(key)
+            logger.debug(f'No match for {key=}. Returning {self.get_default(key)=}')
+        else:
+
+            match_ids = {name: v for name, v in match.groupdict().items() if v}
+            rule_ids = {
+                int(id.removeprefix(self.PREFIX_GROUP))
+                for id in match_ids.keys() if id.startswith(self.PREFIX_GROUP)
+            }
+
+            if len(rule_ids) != 1:
+                msg = f'Multiple group matches: {rule_ids}'
+                raise ValueError(msg)
+
+            rule_id = next(iter(rule_ids))
+            rule = self.items[rule_id]
+
+            logger.debug(f'Matched using {rule_id=}: {rule.source=}')
+
+            if isinstance(rule.target, Key):
+                value = rule.target.transform(match)
+            else:
+                value = rule.target
+
+            logger.debug(f'Transformed using {rule_id=}: {key=} → {value=}')
+
+        return value
+
+    def get_recursive(self, key: Key) -> Key | Any:
+        """
+
+        Lookup the provided key by continuously applying transforms until no changes are made
+        or a circular loop is detected.
+
+        """
+        history = []
+        previous = key
+
+        def get_history_str():
+            return join(history, sep=Constants.ARROW_SEP)
+
+        while True:
+            if previous in history:
+                history.append(previous)
+                msg = f'Loop detected on node "{previous}": {get_history_str()}'
+                raise RewriteCircularLoopError(msg)
+
+            history.append(previous)
+            new = previous
+            new = self.get_one(new)
+            if new == previous:
+                break
+            previous = new
+
+            if not isinstance(new, Key):
+                history.append(previous)
+                break
+
+        if len(history) == 1:
+            history_str = 'No transforms performed.'
+        else:
+            history_str = get_history_str()
+        logger.debug(f'Finished transforming: {history_str}')
+
+        return previous
+
+
+if __name__ == '__main__':
+    ...

fmtr/tools/pdf_tools.py

@@ -1,6 +1,7 @@
+from typing import List, Tuple, Dict, Any, Self
+
 import pymupdf as pm
 import pymupdf4llm
-from typing import List, Tuple, Dict, Any, Self
 
 from fmtr.tools import data_modelling_tools
 
@@ -179,6 +180,43 @@ class Document(pm.Document):
         """
         return pymupdf4llm.to_markdown(self, **kwargs)
 
+    def to_text_pages(self) -> List[str]:
+        """
+
+        Simple text output per-page.
+
+        """
+        lines = []
+        for page in self:
+            text = page.get_text()
+            lines.append(text)
+
+        return lines
+
+    def to_text(self) -> str:
+        """
+
+        Simple text output.
+
+        """
+
+        text = '\n'.join(self.to_text_pages())
+        return text
+
+    def split(self) -> List[Self]:
+        """
+
+        Split pages into individual documents.
+
+        """
+
+        documents = []
+        for i, page in enumerate(self, start=1):
+            document = self.__class__()
+            document.insert_pdf(self, from_page=i, to_page=i)
+            documents.append(document)
+
+        return documents
 
 if __name__ == '__main__':
     from fmtr.tools.path_tools import Path

fmtr/tools/settings_tools.py

@@ -1,10 +1,30 @@
+from typing import ClassVar, Any
+
 from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, YamlConfigSettingsSource, EnvSettingsSource, CliSettingsSource
-from typing import ClassVar
 
-from fmtr.tools.path_tools import PackagePaths
+from fmtr.tools import Constants
+from fmtr.tools.data_modelling_tools import CliRunMixin
+from fmtr.tools.path_tools import PackagePaths, Path
+
+
+class YamlScriptConfigSettingsSource(YamlConfigSettingsSource):
+    """
+
+    Customer source for reading YAML *Script* (as opposed to plain YAML) configuration files.
+
+    """
+
+    def _read_file(self, file_path: Path) -> dict[str, Any]:
+        """
+
+        Use our own Path class to read YAML Script.
+
+        """
+        data = Path(file_path).read_yaml() or {}
+        return data
 
 
-class Base(BaseSettings):
+class Base(BaseSettings, CliRunMixin):
     """
 
     Base class for settings configuration using Pydantic BaseSettings.
@@ -13,6 +33,7 @@ class Base(BaseSettings):
 
     """
 
+    ENV_NESTED_DELIMITER: ClassVar = Constants.ENV_NESTED_DELIMITER
     paths: ClassVar = PackagePaths()
 
     @classmethod
@@ -33,8 +54,8 @@ class Base(BaseSettings):
         sources = (
             init_settings,
             CliSettingsSource(settings_cls, cli_parse_args=True),
-            EnvSettingsSource(settings_cls, env_prefix=cls.get_env_prefix()),
-            YamlConfigSettingsSource(settings_cls, yaml_file=cls.paths.settings),
+            EnvSettingsSource(settings_cls, env_prefix=cls.get_env_prefix(), env_nested_delimiter=cls.ENV_NESTED_DELIMITER),
+            YamlScriptConfigSettingsSource(settings_cls, yaml_file=cls.paths.settings),
         )
 
         return sources
@@ -51,7 +72,7 @@ class Base(BaseSettings):
         else:
             stem = f'{cls.paths.name}'
 
-        prefix = f'{stem}_'.upper()
+        prefix = f'{stem}{cls.ENV_NESTED_DELIMITER}'.upper()
         return prefix
 
     @property

fmtr/tools/setup_tools/__init__.py

@@ -0,0 +1,8 @@
+from fmtr.tools.import_tools import MissingExtraMockModule
+
+from fmtr.tools.setup_tools.setup_tools import Setup, SetupPaths, Dependencies, Tools
+
+try:
+    from setuptools import find_namespace_packages, find_packages, setup as setup_setuptools
+except ModuleNotFoundError as exception:
+    find_namespace_packages = find_packages = setup_setuptools = MissingExtraMockModule('setup', exception)