fmtr.tools 1.3.48__tar.gz → 1.3.50__tar.gz

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.48
+Version: 1.3.50
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -81,16 +81,19 @@ Requires-Dist: pandas; extra == "semantic"
 Requires-Dist: tabulate; extra == "semantic"
 Requires-Dist: openpyxl; extra == "semantic"
 Requires-Dist: odfpy; extra == "semantic"
+Requires-Dist: deepdiff; extra == "semantic"
 Provides-Extra: metric
 Requires-Dist: pandas; extra == "metric"
 Requires-Dist: tabulate; extra == "metric"
 Requires-Dist: openpyxl; extra == "metric"
 Requires-Dist: odfpy; extra == "metric"
+Requires-Dist: deepdiff; extra == "metric"
 Provides-Extra: tabular
 Requires-Dist: pandas; extra == "tabular"
 Requires-Dist: tabulate; extra == "tabular"
 Requires-Dist: openpyxl; extra == "tabular"
 Requires-Dist: odfpy; extra == "tabular"
+Requires-Dist: deepdiff; extra == "tabular"
 Provides-Extra: html
 Requires-Dist: html2text; extra == "html"
 Provides-Extra: interface
@@ -151,67 +154,68 @@ Provides-Extra: db-document
 Requires-Dist: beanie[odm]; extra == "db-document"
 Requires-Dist: motor; extra == "db-document"
 Provides-Extra: all
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
-Requires-Dist: logfire; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: pydantic-extra-types; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
 Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: motor; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
 Requires-Dist: json_repair; extra == "all"
+Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
 Requires-Dist: flet-video; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: motor; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
 Requires-Dist: pycountry; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
+Requires-Dist: logfire; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
 Requires-Dist: beanie[odm]; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
 Requires-Dist: httpx; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: faker; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: html2text; extra == "all"
+Requires-Dist: deepdiff; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: torchvision; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
 Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
-Requires-Dist: diskcache; extra == "all"
-Requires-Dist: peft; extra == "all"
+Requires-Dist: faker; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
 Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: pandas; extra == "all"
 Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
-Requires-Dist: pydantic-extra-types; extra == "all"
-Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
 Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: pydantic; extra == "all"
 Requires-Dist: odfpy; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
 Requires-Dist: cachetools; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: playwright; extra == "all"
 Requires-Dist: uvicorn[standard]; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: tabulate; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/fmtr/tools/database_tools/document.py

@@ -1,8 +1,9 @@
-import beanie
 from functools import cached_property
-from motor.motor_asyncio import AsyncIOMotorClient
 from typing import List
 
+import beanie
+from motor.motor_asyncio import AsyncIOMotorClient
+
 from fmtr.tools.constants import Constants
 
 
@@ -22,7 +23,7 @@ class Client:
         self.port = port
         self.documents = documents
 
-        self.client = AsyncIOMotorClient(self.uri)
+        self.client = AsyncIOMotorClient(self.uri, tz_aware=True)
         self.db = self.client[self.name]
 
     @cached_property

fmtr_tools-1.3.50/fmtr/tools/datetime_tools.py

@@ -0,0 +1,31 @@
+from datetime import datetime, timezone
+from functools import lru_cache
+
+
+def now() -> datetime:
+    """
+
+    Now UTC
+
+    """
+    return datetime.now(tz=timezone.utc)
+
+
+@lru_cache
+def min() -> datetime:
+    """
+
+    Min UTC
+
+    """
+    return now().min
+
+
+@lru_cache
+def max():
+    """
+
+    Max UTC
+
+    """
+    return now().max

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/fmtr/tools/iterator_tools.py

@@ -64,3 +64,12 @@ def strip_none(*items):
 
     """
     return [item for item in items if not is_none(item)]
+
+
+def dedupe(items):
+    """
+
+    Deduplicate a list of items, retaining order
+
+    """
+    return list(dict.fromkeys(items))

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/fmtr/tools/path_tools/path_tools.py

@@ -145,6 +145,32 @@ class Path(type(Path())):
         """
         return self.mkdir(parents=True, exist_ok=True)
 
+    def with_suffix(self, suffix: str) -> 'Path':
+        """
+
+        Pathlib doesn't add a dot prefix, but then errors if you don't provide one, which feels rather obnoxious.
+
+        """
+        if not suffix.startswith('.'):
+            suffix = f'.{suffix}'
+        return super().with_suffix(suffix)
+
+    def get_conversion_path(self, suffix: str) -> 'Path':
+        """
+
+        Fetch the equivalent path for a different format in the standard conversion directory structure.
+        .../xyz/filename.xyx -> ../abc/filename.abc
+
+        """
+
+        old_dir = self.parent.name
+
+        if old_dir != self.suffix.removeprefix('.'):
+            raise ValueError(f"Expected parent directory '{old_dir}' to match file extension '{suffix}'")
+
+        new = self.parent.parent / suffix / f'{self.stem}.{suffix}'
+        return new
+
     @property
     def exist(self):
         """

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/fmtr/tools/pdf_tools.py

@@ -179,6 +179,20 @@ class Document(pm.Document):
         """
         return pymupdf4llm.to_markdown(self, **kwargs)
 
+    def to_text(self):
+        """
+
+        Simple text output.
+
+        """
+        lines = []
+        for page in self:
+            text = page.get_text()
+            lines.append(text)
+
+        text = '\n'.join(lines)
+        return text
+
 
 if __name__ == '__main__':
     from fmtr.tools.path_tools import Path

fmtr_tools-1.3.50/fmtr/tools/tabular_tools.py

@@ -0,0 +1,68 @@
+import deepdiff
+import numpy as np
+import pandas as pd
+
+from fmtr.tools.iterator_tools import dedupe
+
+Table = DataFrame = pd.DataFrame
+Series = pd.Series
+
+nan = np.nan
+
+CONCAT_HORIZONTALLY = 1
+CONCAT_VERTICALLY = 0
+
+
+def normalize_nan(df, value=np.nan):
+    return df.replace({pd.NA: value, None: value, np.nan: value})
+
+
+class Differ:
+    """
+
+    Diff two dataframes via DeepDiff, after shape normalization, datatype simplification, etc.
+
+    """
+
+    def __init__(self, left: Table, right: Table):
+
+        self.cols = dedupe(left.columns.tolist() + right.columns.tolist())
+        self.rows = dedupe(left.index.values.tolist() + right.index.values.tolist())
+        self.left = self.process(left)
+        self.right = self.process(right)
+        self.dfs = [self.left, self.right]
+
+    def process(self, df: Table) -> Table:
+        """
+
+        Ensure same rows/columns, plus simplify datatypes via JSON round-robin.
+
+        """
+
+        df_rows = set(df.index.values.tolist())
+        for row in self.rows:
+            if row in df_rows:
+                continue
+            df.loc[len(df)] = None
+
+        df_cols = set(df.columns.tolist())
+        for col in self.cols:
+            if col in df_cols:
+                continue
+            df[col] = None
+
+        df = pd.read_json(df.to_json(date_format='iso'))
+        df = normalize_nan(df, value=None)
+
+        return df
+
+    def get_diff(self) -> deepdiff.DeepDiff:
+        """
+
+        Cast to dicts and get diff
+
+        """
+
+        dicts = [df.to_dict(orient='index') for df in self.dfs]
+        diff = deepdiff.DeepDiff(*dicts, ignore_order=True)
+        return diff

fmtr_tools-1.3.50/fmtr/tools/version

@@ -0,0 +1 @@
+1.3.50

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/fmtr.tools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.48
+Version: 1.3.50
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -81,16 +81,19 @@ Requires-Dist: pandas; extra == "semantic"
 Requires-Dist: tabulate; extra == "semantic"
 Requires-Dist: openpyxl; extra == "semantic"
 Requires-Dist: odfpy; extra == "semantic"
+Requires-Dist: deepdiff; extra == "semantic"
 Provides-Extra: metric
 Requires-Dist: pandas; extra == "metric"
 Requires-Dist: tabulate; extra == "metric"
 Requires-Dist: openpyxl; extra == "metric"
 Requires-Dist: odfpy; extra == "metric"
+Requires-Dist: deepdiff; extra == "metric"
 Provides-Extra: tabular
 Requires-Dist: pandas; extra == "tabular"
 Requires-Dist: tabulate; extra == "tabular"
 Requires-Dist: openpyxl; extra == "tabular"
 Requires-Dist: odfpy; extra == "tabular"
+Requires-Dist: deepdiff; extra == "tabular"
 Provides-Extra: html
 Requires-Dist: html2text; extra == "html"
 Provides-Extra: interface
@@ -151,67 +154,68 @@ Provides-Extra: db-document
 Requires-Dist: beanie[odm]; extra == "db-document"
 Requires-Dist: motor; extra == "db-document"
 Provides-Extra: all
-Requires-Dist: filetype; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
-Requires-Dist: logfire; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
+Requires-Dist: regex; extra == "all"
+Requires-Dist: distributed; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: pydantic-extra-types; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
 Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: motor; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
 Requires-Dist: json_repair; extra == "all"
+Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: pydantic-settings; extra == "all"
 Requires-Dist: flet-video; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: pyyaml; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: motor; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: bokeh; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
 Requires-Dist: pycountry; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
+Requires-Dist: logfire; extra == "all"
+Requires-Dist: httpx_retries; extra == "all"
 Requires-Dist: beanie[odm]; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: pydantic-settings; extra == "all"
 Requires-Dist: httpx; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: faker; extra == "all"
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: html2text; extra == "all"
+Requires-Dist: deepdiff; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: filetype; extra == "all"
+Requires-Dist: torchvision; extra == "all"
+Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
 Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: pydantic; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: distributed; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
-Requires-Dist: diskcache; extra == "all"
-Requires-Dist: peft; extra == "all"
+Requires-Dist: faker; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
 Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: pandas; extra == "all"
 Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: regex; extra == "all"
-Requires-Dist: google-auth; extra == "all"
-Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: pymupdf4llm; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: httpx_retries; extra == "all"
-Requires-Dist: pydantic-extra-types; extra == "all"
-Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
 Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: sre_yield; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: pydantic; extra == "all"
 Requires-Dist: odfpy; extra == "all"
+Requires-Dist: pymupdf4llm; extra == "all"
 Requires-Dist: cachetools; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: fastapi; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: playwright; extra == "all"
 Requires-Dist: uvicorn[standard]; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
+Requires-Dist: html2text; extra == "all"
+Requires-Dist: tabulate; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/fmtr.tools.egg-info/requires.txt

@@ -18,67 +18,68 @@ pydantic-ai[logfire,openai]
 ollama
 
 [all]
-filetype
-flet-webview
-pydevd-pycharm~=251.25410.159
-logfire
+flet[all]
+regex
+distributed
+tokenizers
+pydantic-extra-types
+transformers[sentencepiece]
 sentence_transformers
-motor
-torchvision
-google-auth-oauthlib
 json_repair
+Unidecode
+google-auth
+pydantic-settings
 flet-video
+fastapi
+pyyaml
+huggingface_hub
+dnspython[doh]
+motor
+semver
+openai
+logfire[fastapi]
+pandas
+diskcache
+bokeh
+google-api-python-client
+deepmerge
+torchaudio
+contexttimer
 pycountry
-transformers[sentencepiece]
+pymupdf
+logfire
+httpx_retries
 beanie[odm]
-yamlscript
-appdirs
-pydantic-settings
 httpx
-contexttimer
-bokeh
-google-auth-httplib2
-huggingface_hub
-faker
-dnspython[doh]
-html2text
+deepdiff
+ollama
+filetype
+torchvision
+sre_yield
+appdirs
+setuptools
+pydevd-pycharm~=251.25410.159
+yamlscript
 tinynetrc
-pydantic
-torchaudio
-distributed
-flet[all]
-pydantic-ai[logfire,openai]
-diskcache
-peft
+faker
+flet-webview
 pytest-cov
-pandas
 logfire[httpx]
-regex
-google-auth
-pyyaml
-pymupdf4llm
-ollama
-openai
-tokenizers
-pymupdf
-httpx_retries
-pydantic-extra-types
-Unidecode
+dask[bag]
 openpyxl
-sre_yield
+playwright
+google-auth-httplib2
+pydantic-ai[logfire,openai]
+peft
+docker
+pydantic
 odfpy
+pymupdf4llm
 cachetools
-dask[bag]
-deepmerge
-fastapi
-google-api-python-client
-semver
-tabulate
-logfire[fastapi]
-setuptools
-docker
-playwright
 uvicorn[standard]
+google-auth-oauthlib
+html2text
+tabulate
 
 [api]
 fastapi
@@ -161,6 +162,7 @@ pandas
 tabulate
 openpyxl
 odfpy
+deepdiff
 
 [netrc]
 tinynetrc
@@ -201,6 +203,7 @@ pandas
 tabulate
 openpyxl
 odfpy
+deepdiff
 
 [sets]
 pydantic-settings
@@ -221,6 +224,7 @@ pandas
 tabulate
 openpyxl
 odfpy
+deepdiff
 
 [test]
 pytest-cov

{fmtr_tools-1.3.48 → fmtr_tools-1.3.50}/setup.py

@@ -26,7 +26,7 @@ DEPENDENCIES = {
     'json-fix': ['json_repair'],
     'semantic': ['sentence_transformers', 'metric'],
     'metric': ['tabular'],
-    'tabular': ['pandas', 'tabulate', 'openpyxl', 'odfpy'],
+    'tabular': ['pandas', 'tabulate', 'openpyxl', 'odfpy', 'deepdiff'],
     'html': ['html2text'],
     'interface': ['flet[all]', 'flet-video', 'flet-webview'],
     'google.api': ['google-auth', 'google-auth-oauthlib', 'google-auth-httplib2', 'google-api-python-client'],

fmtr_tools-1.3.48/fmtr/tools/datetime_tools.py

@@ -1,5 +0,0 @@
-from datetime import datetime, timezone
-
-
-def now():
-    return datetime.now(tz=timezone.utc)

fmtr_tools-1.3.48/fmtr/tools/tabular_tools.py

@@ -1,7 +0,0 @@
-import pandas as pd
-
-Table = DataFrame = pd.DataFrame
-Series = pd.Series
-
-CONCAT_HORIZONTALLY = 1
-CONCAT_VERTICALLY = 0

fmtr_tools-1.3.48/fmtr/tools/version

@@ -1 +0,0 @@
-1.3.48