fmtr.tools 1.2.6__tar.gz → 1.2.7__tar.gz

{fmtr_tools-1.2.6 → fmtr_tools-1.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.2.6
+Version: 1.2.7
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/tools
 Author: Frontmatter
@@ -130,60 +130,60 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
 Requires-Dist: Unidecode; extra == "all"
 Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: faker; extra == "all"
 Requires-Dist: semver; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: bokeh; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: regex; extra == "all"
 Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: diskcache; extra == "all"
 Requires-Dist: peft; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: pydevd-pycharm; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: html2text; extra == "all"
-Requires-Dist: faker; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: pydantic; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: logfire; extra == "all"
 Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: openai; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: ollama; extra == "all"
 Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: tabulate; extra == "all"
+Requires-Dist: pydevd-pycharm; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: bokeh; extra == "all"
 Requires-Dist: fastapi; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
 Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: pandas; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
 Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: logfire; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: torchvision; extra == "all"
 Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

fmtr_tools-1.2.7/fmtr/tools/pattern_tools.py

@@ -0,0 +1,232 @@
+import regex as re
+from dataclasses import dataclass, asdict
+from functools import cached_property
+from typing import List, Any
+
+from fmtr.tools.logging_tools import logger
+from fmtr.tools.string_tools import join
+
+
+class RewriteCircularLoopError(Exception):
+    """
+
+    Circular loop error
+
+    """
+
+
+MASK_GROUP = '(?:{pattern})'
+MASK_NAMED = r"(?P<{key}>{pattern})"
+
+
+def alt(*patterns):
+    patterns = sorted(patterns, key=len, reverse=True)
+    pattern = '|'.join(patterns)
+    pattern = MASK_GROUP.format(pattern=pattern)
+    return pattern
+
+
+
+
+
+
+@dataclass
+class Key:
+    RECORD_SEP = '␞'
+
+    def flatten(self, data):
+        """
+
+        Flatten/serialise dictionary data
+
+        """
+        pairs = [f'{value}' for key, value in data.items()]
+        string = self.RECORD_SEP.join(pairs)
+        return string
+
+    @cached_property
+    def pattern(self):
+        """
+
+        Serialise to pattern
+
+        """
+        data = {key: MASK_NAMED.format(key=key, pattern=value) for key, value in asdict(self).items()}
+        pattern = self.flatten(data)
+        return pattern
+
+    @cached_property
+    def rx(self):
+        """
+
+        Compile to Regular Expression
+
+        """
+        return re.compile(self.pattern)
+
+    @cached_property
+    def string(self):
+        """
+
+        Serialise to string
+
+        """
+        string = self.flatten(asdict(self))
+        return string
+
+    def transform(self, match: re.Match):
+        """
+
+        Transform match object into a new object of the same type.
+
+        """
+        groupdict = match.groupdict()
+        data = {key: value.format(**groupdict) for key, value in asdict(self).items()}
+        obj = self.__class__(**data)
+        return obj
+
+
+@dataclass
+class Item:
+    """
+
+    Key-value pair
+
+    """
+    key: Key
+    value: Key
+
+@dataclass
+class Mapper:
+    """
+    
+    Pattern-based, dictionary-like mapper.
+    Compiles a single regex pattern from a list of rules, and determines which rule matched.
+    It supports initialization from structured rule data, execution of a single lookup pass, and
+    recursive lookups until a stable state is reached.
+
+    """
+    PREFIX_GROUP = '__'
+    items: List[Item]
+    default: Any = None
+    is_recursive: bool = False
+
+    @cached_property
+    def pattern(self):
+        """
+        
+        Provides a dynamically generated regex pattern based on the rules provided.
+
+        """
+        patterns = [
+            MASK_NAMED.format(key=f'{self.PREFIX_GROUP}{i}', pattern=item.key.pattern)
+            for i, item in enumerate(self.items)
+        ]
+        pattern = alt(*patterns)
+        return pattern
+
+    @cached_property
+    def rx(self):
+        """
+
+        Regex object.
+
+        """
+        return re.compile(self.pattern)
+
+    def get_default(self, key: Key):
+        if self.is_recursive:
+            return key
+        else:
+            return self.default
+
+    def get(self, key: Key) -> Key:
+        """
+
+        Use recursive or single lookup pass, depending on whether recursive lookups have been specified.
+
+        """
+        if self.is_recursive:
+            return self.get_recursive(key)
+        else:
+            return self.get_one(key)
+
+    def get_one(self, key: Key):
+        """
+
+        Single lookup pass.
+        Lookup the source string based on the matching rule.
+
+        """
+
+        match = self.rx.fullmatch(key.string)
+
+        if not match:
+            value = self.get_default(key)
+            logger.debug(f'No match for {key=}.')
+        else:
+
+            match_ids = {name: v for name, v in match.groupdict().items() if v}
+            rule_ids = {
+                int(id.removeprefix(self.PREFIX_GROUP))
+                for id in match_ids.keys() if id.startswith(self.PREFIX_GROUP)
+            }
+
+            if len(rule_ids) != 1:
+                msg = f'Multiple group matches: {rule_ids}'
+                raise ValueError(msg)
+
+            rule_id = next(iter(rule_ids))
+            rule = self.items[rule_id]
+
+            if isinstance(rule.value, Key):
+                value = rule.value.transform(match)
+            else:
+                value = rule.value
+
+            logger.debug(f'Matched using {rule_id=}: {key=} → {value=}')
+
+        return value
+
+    def get_recursive(self, key: Key) -> Key:
+        """
+
+        Lookup the provided text by continuously applying lookup rules until no changes are made
+        or a circular loop is detected.
+
+        """
+        history = []
+        previous = key
+
+        def get_history_str():
+            return join(history, sep=' → ')
+
+        with logger.span(f'Matching {key=}...'):
+            while True:
+                if previous in history:
+                    history.append(previous)
+                    msg = f'Loop detected on node "{previous}": {get_history_str()}'
+                    raise RewriteCircularLoopError(msg)
+
+                history.append(previous)
+
+                new = previous
+
+                new = self.get_one(new)
+
+                if new == previous:
+                    break
+
+                previous = new
+
+        if len(history) == 1:
+            history_str = 'No matching performed.'
+        else:
+            history_str = get_history_str()
+        logger.debug(f'Finished matching: {history_str}')
+
+        return previous
+
+
+if __name__ == '__main__':
+    ...

fmtr_tools-1.2.7/fmtr/tools/version

@@ -0,0 +1 @@
+1.2.7

{fmtr_tools-1.2.6 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.2.6
+Version: 1.2.7
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/tools
 Author: Frontmatter
@@ -130,60 +130,60 @@ Requires-Dist: logfire[httpx]; extra == "http"
 Provides-Extra: setup
 Requires-Dist: setuptools; extra == "setup"
 Provides-Extra: all
-Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
 Requires-Dist: Unidecode; extra == "all"
 Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: faker; extra == "all"
 Requires-Dist: semver; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: bokeh; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: regex; extra == "all"
 Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: diskcache; extra == "all"
 Requires-Dist: peft; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: pydevd-pycharm; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: json_repair; extra == "all"
-Requires-Dist: html2text; extra == "all"
-Requires-Dist: faker; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: httpx; extra == "all"
+Requires-Dist: pydantic; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: logfire; extra == "all"
 Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: openai; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: pytest-cov; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: json_repair; extra == "all"
+Requires-Dist: ollama; extra == "all"
 Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: tabulate; extra == "all"
+Requires-Dist: pydevd-pycharm; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: bokeh; extra == "all"
 Requires-Dist: fastapi; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
 Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: flet-video; extra == "all"
-Requires-Dist: pandas; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
 Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: logfire; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: torchvision; extra == "all"
 Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

fmtr_tools-1.2.6/fmtr/tools/pattern_tools.py

@@ -1,175 +0,0 @@
-import regex as re
-from dataclasses import dataclass
-from functools import cached_property
-from typing import List
-
-from fmtr.tools.logging_tools import logger
-
-
-class RewriteCircularLoopError(Exception):
-    """
-
-    Circular loop error
-
-    """
-
-
-@dataclass
-class Rewrite:
-    """
-    Represents a single rule for pattern matching and target string replacement.
-
-    This class is used to define a rule with a pattern and a target string.
-    The `pattern` is a regular expression used to identify matches in input text.
-    The `target` allows rewriting the identified matches with a formatted string.
-    It provides properties for generating a unique identifier for use as a regex group name and compiling the provided pattern into a regular expression object.
-
-    """
-    pattern: str
-    target: str
-
-    @cached_property
-    def id(self):
-        """
-        
-        Regex group name.
-        
-        """
-        return f'id{abs(hash(self.pattern))}'
-
-    @cached_property
-    def rx(self):
-        """
-
-        Regex object.
-
-        """
-        return re.compile(self.pattern)
-
-    def apply(self, match: re.Match):
-        """
-
-        Rewrite using the target string and match groups.
-
-        """
-        target = self.target.format(**match.groupdict())
-        return target
-
-
-@dataclass
-class Rewriter:
-    """
-    
-    Represents a Rewriter class that handles pattern matching, rule application, and text rewriting.
-    Compiles a single regex pattern from a list of rules, and determines which rule matched.
-    It supports initialization from structured rule data, execution of a single rewrite pass, and
-    recursive rewriting until a stable state is reached.
-
-    """
-    rules: List[Rewrite]
-
-    @cached_property
-    def pattern(self):
-        """
-        
-        Provides a dynamically generated regex pattern based on the rules provided.
-
-        """
-        patterns = [fr"(?P<{rule.id}>{rule.pattern})" for rule in self.rules]
-        sorted(patterns, key=len, reverse=True)
-        pattern = '|'.join(patterns)
-        return pattern
-
-    @cached_property
-    def rule_lookup(self):
-        """
-
-        Dictionary mapping rule identifiers to their corresponding rules.
-        """
-
-        return {rule.id: rule for rule in self.rules}
-
-    @cached_property
-    def rx(self):
-        """
-
-        Regex object.
-
-        """
-        return re.compile(self.pattern)
-
-    def rewrite_pass(self, source: str):
-        """
-
-        Single rewrite pass.
-        Rewrites the provided source string based on the matching rule.
-
-        """
-
-        match = self.rx.fullmatch(source)
-
-        if not match:
-            return source
-
-        match_ids = {k: v for k, v in match.groupdict().items() if v}
-        match_id = match_ids & self.rule_lookup.keys()
-
-        if len(match_id) != 1:
-            msg = f'Multiple group matches: {match_id}'
-            raise ValueError(msg)
-
-        match_id = next(iter(match_id))
-        rule = self.rule_lookup[match_id]
-        target = rule.apply(match)
-
-        logger.debug(f'Rewrote using {match_id=}: {source=} -> {target=}')
-
-        return target
-
-    def rewrite(self, source: str) -> str:
-        """
-
-        Rewrites the provided text by continuously applying rewrite rules until no changes are made
-        or a circular loop is detected.
-
-        """
-        history = []
-        previous = source
-
-        def get_history_str():
-            return ' -> '.join(history)
-
-        with logger.span(f'Rewriting "{source}"...'):
-            while True:
-                if previous in history:
-                    history.append(previous)
-                    msg = f'Loop detected on node "{previous}": {get_history_str()}'
-                    raise RewriteCircularLoopError(msg)
-
-                history.append(previous)
-
-                new = previous
-
-                new = self.rewrite_pass(new)
-
-                if new == previous:
-                    break
-
-                previous = new
-
-        if len(history) == 1:
-            history_str = 'No rewrites performed.'
-        else:
-            history_str = get_history_str()
-        logger.debug(f'Finished rewriting: {history_str}')
-
-        return previous
-
-    @classmethod
-    def from_data(cls, data):
-        rules = [Rewrite(*pair) for pair in data.items()]
-        self = cls(rules=rules)
-        return self
-
-
-

fmtr_tools-1.2.6/fmtr/tools/version

@@ -1 +0,0 @@
-1.2.6

{fmtr_tools-1.2.6 → fmtr_tools-1.2.7}/fmtr.tools.egg-info/requires.txt

@@ -15,60 +15,60 @@ pydantic-ai[logfire,openai]
 ollama
 
 [all]
-google-api-python-client
+distributed
+diskcache
+sre_yield
+httpx_retries
+filetype
+logfire[httpx]
+pymupdf4llm
 Unidecode
 uvicorn[standard]
-huggingface_hub
-logfire[fastapi]
-pymupdf4llm
+faker
 semver
-pymupdf
-bokeh
+pyyaml
+regex
 contexttimer
-diskcache
 peft
-dask[bag]
-pydevd-pycharm
-transformers[sentencepiece]
-regex
-json_repair
-html2text
-faker
+flet-webview
+pydantic-settings
+deepmerge
+httpx
+pydantic
+pymupdf
+google-auth
+google-api-python-client
+logfire
 openpyxl
-appdirs
-logfire[httpx]
-openai
+html2text
+pytest-cov
+docker
+google-auth-oauthlib
+json_repair
+ollama
 sentence_transformers
-pydantic-ai[logfire,openai]
+pandas
+setuptools
+logfire[fastapi]
+tabulate
+pydevd-pycharm
+huggingface_hub
+flet-video
+dask[bag]
+google-auth-httplib2
+bokeh
 fastapi
-torchvision
-ollama
-tokenizers
 dnspython[doh]
-flet-video
-pandas
+transformers[sentencepiece]
 yamlscript
-filetype
-logfire
-tabulate
-google-auth-oauthlib
+torchaudio
+pydantic-ai[logfire,openai]
+torchvision
 flet[all]
-pydantic-settings
+openai
 tinynetrc
-deepmerge
-distributed
-flet-webview
-setuptools
-pyyaml
-sre_yield
-google-auth
-torchaudio
-pydantic
-google-auth-httplib2
-docker
-httpx
-pytest-cov
-httpx_retries
+appdirs
+tokenizers
 
 [api]
 fastapi