fmtr.tools 1.2.6__tar.gz → 1.3.0__tar.gz

{fmtr_tools-1.2.6 → fmtr_tools-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.2.6
+Version: 1.3.0
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/tools
 Author: Frontmatter
@@ -130,60 +130,60 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: pydantic; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: logfire; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: pydevd-pycharm; extra == "all"
+Requires-Dist: torchvision; extra == "all"
 Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
+Requires-Dist: faker; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
+Requires-Dist: tinynetrc; extra == "all"
 Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
 Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: semver; extra == "all"
 Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: diskcache; extra == "all"
-Requires-Dist: peft; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: pydevd-pycharm; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: html2text; extra == "all"
-Requires-Dist: faker; extra == "all"
-Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: appdirs; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: pandas; extra == "all"
 Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
 Requires-Dist: openai; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
 Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: peft; extra == "all"
 Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
 Requires-Dist: fastapi; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: logfire; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
 Requires-Dist: tabulate; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
 Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: tinynetrc; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
 Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: html2text; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.2.6 → fmtr_tools-1.3.0}/fmtr/tools/dns_tools/server.py

@@ -66,7 +66,7 @@ class ServerBaseDoHProxy(ServerBasePlain):
 
         request = exchange.request
 
-        with logger.span(f'Handling request for {request.name_text} from {exchange.client}...'):
+        with logger.span(f'Handling request ID {request.message.id} for {request.name_text} from {exchange.client}...'):
 
             if not request.is_valid:
                 raise ValueError(f'Only one question per request is supported. Got {len(request.question)} questions.')

fmtr_tools-1.3.0/fmtr/tools/pattern_tools.py

@@ -0,0 +1,232 @@
+import regex as re
+from dataclasses import dataclass, asdict
+from functools import cached_property
+from typing import List, Any
+
+from fmtr.tools.logging_tools import logger
+from fmtr.tools.string_tools import join
+
+
+class RewriteCircularLoopError(Exception):
+    """
+
+    Circular loop error
+
+    """
+
+
+MASK_GROUP = '(?:{pattern})'
+MASK_NAMED = r"(?P<{key}>{pattern})"
+
+
+def alt(*patterns):
+    patterns = sorted(patterns, key=len, reverse=True)
+    pattern = '|'.join(patterns)
+    pattern = MASK_GROUP.format(pattern=pattern)
+    return pattern
+
+
+
+
+
+
+@dataclass
+class Key:
+    RECORD_SEP = '␞'
+
+    def flatten(self, data):
+        """
+
+        Flatten/serialise dictionary data
+
+        """
+        pairs = [f'{value}' for key, value in data.items()]
+        string = self.RECORD_SEP.join(pairs)
+        return string
+
+    @cached_property
+    def pattern(self):
+        """
+
+        Serialise to pattern
+
+        """
+        data = {key: MASK_NAMED.format(key=key, pattern=value) for key, value in asdict(self).items()}
+        pattern = self.flatten(data)
+        return pattern
+
+    @cached_property
+    def rx(self):
+        """
+
+        Compile to Regular Expression
+
+        """
+        return re.compile(self.pattern)
+
+    @cached_property
+    def string(self):
+        """
+
+        Serialise to string
+
+        """
+        string = self.flatten(asdict(self))
+        return string
+
+    def transform(self, match: re.Match):
+        """
+
+        Transform match object into a new object of the same type.
+
+        """
+        groupdict = match.groupdict()
+        data = {key: value.format(**groupdict) for key, value in asdict(self).items()}
+        obj = self.__class__(**data)
+        return obj
+
+
+@dataclass
+class Item:
+    """
+
+    Key-value pair
+
+    """
+    key: Key
+    value: Key
+
+@dataclass
+class Mapper:
+    """
+    
+    Pattern-based, dictionary-like mapper.
+    Compiles a single regex pattern from a list of rules, and determines which rule matched.
+    It supports initialization from structured rule data, execution of a single lookup pass, and
+    recursive lookups until a stable state is reached.
+
+    """
+    PREFIX_GROUP = '__'
+    items: List[Item]
+    default: Any = None
+    is_recursive: bool = False
+
+    @cached_property
+    def pattern(self):
+        """
+        
+        Provides a dynamically generated regex pattern based on the rules provided.
+
+        """
+        patterns = [
+            MASK_NAMED.format(key=f'{self.PREFIX_GROUP}{i}', pattern=item.key.pattern)
+            for i, item in enumerate(self.items)
+        ]
+        pattern = alt(*patterns)
+        return pattern
+
+    @cached_property
+    def rx(self):
+        """
+
+        Regex object.
+
+        """
+        return re.compile(self.pattern)
+
+    def get_default(self, key: Key):
+        if self.is_recursive:
+            return key
+        else:
+            return self.default
+
+    def get(self, key: Key) -> Key:
+        """
+
+        Use recursive or single lookup pass, depending on whether recursive lookups have been specified.
+
+        """
+        if self.is_recursive:
+            return self.get_recursive(key)
+        else:
+            return self.get_one(key)
+
+    def get_one(self, key: Key):
+        """
+
+        Single lookup pass.
+        Lookup the source string based on the matching rule.
+
+        """
+
+        match = self.rx.fullmatch(key.string)
+
+        if not match:
+            value = self.get_default(key)
+            logger.debug(f'No match for {key=}.')
+        else:
+
+            match_ids = {name: v for name, v in match.groupdict().items() if v}
+            rule_ids = {
+                int(id.removeprefix(self.PREFIX_GROUP))
+                for id in match_ids.keys() if id.startswith(self.PREFIX_GROUP)
+            }
+
+            if len(rule_ids) != 1:
+                msg = f'Multiple group matches: {rule_ids}'
+                raise ValueError(msg)
+
+            rule_id = next(iter(rule_ids))
+            rule = self.items[rule_id]
+
+            if isinstance(rule.value, Key):
+                value = rule.value.transform(match)
+            else:
+                value = rule.value
+
+            logger.debug(f'Matched using {rule_id=}: {key=} → {value=}')
+
+        return value
+
+    def get_recursive(self, key: Key) -> Key:
+        """
+
+        Lookup the provided text by continuously applying lookup rules until no changes are made
+        or a circular loop is detected.
+
+        """
+        history = []
+        previous = key
+
+        def get_history_str():
+            return join(history, sep=' → ')
+
+        with logger.span(f'Matching {key=}...'):
+            while True:
+                if previous in history:
+                    history.append(previous)
+                    msg = f'Loop detected on node "{previous}": {get_history_str()}'
+                    raise RewriteCircularLoopError(msg)
+
+                history.append(previous)
+
+                new = previous
+
+                new = self.get_one(new)
+
+                if new == previous:
+                    break
+
+                previous = new
+
+        if len(history) == 1:
+            history_str = 'No matching performed.'
+        else:
+            history_str = get_history_str()
+        logger.debug(f'Finished matching: {history_str}')
+
+        return previous
+
+
+if __name__ == '__main__':
+    ...

{fmtr_tools-1.2.6 → fmtr_tools-1.3.0}/fmtr/tools/setup_tools/setup_tools.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from fnmatch import fnmatch
 from functools import cached_property
 from itertools import chain
-from typing import List, Dict
+from typing import List, Dict, Any, Callable, Optional
 
 from fmtr.tools.constants import Constants
 from fmtr.tools.path_tools import Path
@@ -18,7 +18,6 @@ class SetupPaths(FromCallerMixin):
     """
 
     def __init__(self, path=None, org=Constants.ORG_NAME):
-
         """
 
         Use calling module path as default path, if not otherwise specified.
@@ -31,15 +30,30 @@ class SetupPaths(FromCallerMixin):
         self.repo = Path(path)
 
     @property
-    def readme(self):
+    def readme(self) -> Path:
+        """
+
+        Path of the README file.
+
+        """
         return self.repo / 'README.md'
 
     @property
-    def version(self):
+    def version(self) -> Path:
+        """
+
+        Path of the version file
+
+        """
         return self.path / Constants.FILENAME_VERSION
 
     @cached_property
-    def path(self):
+    def path(self) -> Path:
+        """
+
+        Infer the package path. It should be the only non-excluded package in the repo/org Path.
+
+        """
 
         if self.is_namespace:
             base = self.org
@@ -61,7 +75,12 @@ class SetupPaths(FromCallerMixin):
         return package
 
     @property
-    def org(self):
+    def org(self) -> bool | Path:
+        """
+
+        Get the org path, i.e. the namespace parent directory.
+
+        """
         if not self.org_name:
             return False
         org = self.repo / self.org_name
@@ -70,7 +89,12 @@ class SetupPaths(FromCallerMixin):
         return org
 
     @property
-    def entrypoints(self):
+    def entrypoints(self) -> Path:
+        """
+
+        Path of entrypoints sub-package.
+
+        """
         return self.path / Constants.ENTRYPOINTS_DIR
 
     @property
@@ -83,6 +107,11 @@ class SetupPaths(FromCallerMixin):
 
 
 class Setup(FromCallerMixin):
+    """
+
+    Abstract canonical pacakge setup for setuptools.
+
+    """
     AUTHOR = 'Frontmatter'
     AUTHOR_EMAIL = 'innovative.fowler@mask.pro.fmtr.dev'
 
@@ -93,8 +122,12 @@ class Setup(FromCallerMixin):
     ENTRYPOINT_FUNC_NAME = 'main'
 
     def __init__(self, dependencies, paths=None, org=Constants.ORG_NAME, client=None, do_setup=True, **kwargs):
+        """
 
+        First check if commandline arguments for requirements output exist. If so, print them and return early.
+        Otherwise, continue generating data to pass to setuptools.
 
+        """
         self.kwargs = kwargs
 
         if type(dependencies) is not Dependencies:
@@ -115,11 +148,15 @@ class Setup(FromCallerMixin):
 
         self.client = client
 
-
         if do_setup:
             self.setup()
 
-    def get_requirements_extras(self):
+    def get_requirements_extras(self) -> Optional[List[str]]:
+        """
+
+        Get list of extras from command line arguments.
+
+        """
         if self.REQUIREMENTS_ARG not in sys.argv:
             return None
 
@@ -128,6 +165,11 @@ class Setup(FromCallerMixin):
         return extras
 
     def print_requirements(self):
+        """
+
+        Output flat list of requirements for specified extras
+
+        """
         reqs = []
         reqs += self.dependencies.install
 
@@ -137,12 +179,17 @@ class Setup(FromCallerMixin):
         print(reqs)
 
     @property
-    def console_scripts(self):
+    def console_scripts(self) -> List[str]:
+        """
+
+        Generate console scripts for any modules in the `entrypoints` sub-package.
+
+        """
 
         if not self.paths.entrypoints.exists():
-            return {}
+            return []
 
-        names_mods = [path.stem for path in self.paths.entrypoints.iterdir() if path.is_file() and not path.stem.startswith('_')]
+        names_mods = [path.stem for path in self.paths.entrypoints.iterdir() if path.is_file() and path.name != Constants.INIT_FILENAME]
         command_prefix = self.name.replace('.', self.ENTRYPOINT_COMMAND_SEP)
         command_suffixes = [name_mod.replace(self.ENTRYPOINT_FUNCTION_SEP, self.ENTRYPOINT_COMMAND_SEP) for name_mod in names_mods]
         commands = [f'{command_prefix}-{command_suffix}' for command_suffix in command_suffixes]
@@ -153,35 +200,63 @@ class Setup(FromCallerMixin):
         return console_scripts
 
     @property
-    def name(self):
+    def name(self) -> str:
+        """
+
+        Full library name
+
+        """
         if self.paths.is_namespace:
             return f'{self.paths.org_name}.{self.paths.name}'
         return self.paths.name
 
     @property
-    def author(self):
+    def author(self) -> str:
+        """
+
+        Create appropriate author string
+
+        """
         if self.client:
             return f'{self.AUTHOR} on behalf of {self.client}'
         return self.AUTHOR
 
     @property
-    def copyright(self):
+    def copyright(self) -> str:
+        """
+
+        Create appropriate copyright string
+
+        """
         if self.client:
             return self.client
         return self.AUTHOR
 
     @property
-    def long_description(self):
+    def long_description(self) -> str:
+        """
+
+        Read in README.md
 
+        """
         return self.paths.readme.read_text()
 
     @property
-    def version(self):
+    def version(self) -> str:
+        """
+
+        Read in the version string from file
+
+        """
         return self.paths.version.read_text().strip()
 
     @property
-    def find(self):
+    def find(self) -> Callable:
+        """
+
+        Use the appropriate package finding function from setuptools
 
+        """
         from fmtr.tools import setup
 
         if self.paths.is_namespace:
@@ -190,13 +265,23 @@ class Setup(FromCallerMixin):
             return setup.find_packages
 
     @property
-    def packages(self):
+    def packages(self) -> List[str]:
+        """
+
+        Fetch list of packages excluding canonical paths
+
+        """
         excludes = list(Constants.PACKAGE_EXCLUDE_DIRS) + [f'{name}.*' for name in Constants.PACKAGE_EXCLUDE_DIRS if '*' not in name]
         packages = self.find(where=str(self.paths.repo), exclude=excludes)
         return packages
 
     @property
     def package_dir(self):
+        """
+
+        Needs to be relative apparently as absolute paths break during packaging
+
+        """
         if self.paths.is_namespace:
             return {'': '.'}
         else:
@@ -204,14 +289,29 @@ class Setup(FromCallerMixin):
 
     @property
     def package_data(self):
+        """
+
+        Default package data is just the version file
+
+        """
         return {self.name: [Constants.FILENAME_VERSION]}
 
     @property
-    def url(self):
-        return f'https://github.com/{self.org}/{self.paths.name}'
+    def url(self) -> str:
+        """
+
+        Default to GitHub URL
+
+        """
+        return f'https://github.com/{self.org}/{self.name}'
 
     @property
-    def data(self):
+    def data(self) -> Dict[str, Any]:
+        """
+
+        Generate data for use by setuptools
+
+        """
         data = dict(
             name=self.name,
             version=self.version,
@@ -233,13 +333,30 @@ class Setup(FromCallerMixin):
         return data
 
     def setup(self):
+        """
+
+        Call setuptools.setup using generated data
+
+        """
 
         from fmtr.tools import setup
 
         return setup.setup_setuptools(**self.data)
 
+    def __repr__(self) -> str:
+        """
+
+        Show library name
+
+        """
+        return f'{self.__class__.__name__}("{self.name}")'
 
 class Tools:
+    """
+
+    Helper for downstream libraries to specify lists of `fmtr.tools` extras
+
+    """
     MASK = f'{Constants.LIBRARY_NAME}[{{extras}}]'
 
     def __init__(self, *extras):
@@ -261,7 +378,7 @@ class Dependencies:
     def resolve_values(self, key) -> List[str]:
         """
 
-        Flatten a list of values.
+        Flatten a list of dependencies.
 
         """
         values_resolved = []
@@ -290,7 +407,12 @@ class Dependencies:
         return resolved
 
     @cached_property
-    def install(self):
+    def install(self) -> List[str]:
+        """
+
+        Get install_requires
+
+        """
         if self.INSTALL in self.dependencies:
             return self.resolve_values(self.INSTALL)
         else:
@@ -298,26 +420,4 @@ class Dependencies:
 
 
 if __name__ == '__main__':
-    ds = Dependencies(
-        install=['version', 'yaml'],
-
-        yaml=['yamlscript', 'pyyaml'],
-        logging=['logfire', 'version'],
-        version=['semver', 'av'],
-        av=['av']
-        # Add the rest of your dependencies...
-    )
-
-    ds
-
-    setup = Setup(
-        # client='Acme',
-        dependencies=ds,
-        description='some tools test',
-        console_scripts=dict(
-            cache_hfh='console_script_tools',
-            test=None,
-        )
-    )
-    data = setup.get_data_setup()
-    data
+    ...