fmtr.tools 1.3.48__py3-none-any.whl → 1.3.49__py3-none-any.whl

fmtr/tools/iterator_tools.py

@@ -64,3 +64,12 @@ def strip_none(*items):
 
     """
     return [item for item in items if not is_none(item)]
+
+
+def dedupe(items):
+    """
+
+    Deduplicate a list of items, retaining order
+
+    """
+    return list(dict.fromkeys(items))

fmtr/tools/path_tools/path_tools.py

@@ -145,6 +145,32 @@ class Path(type(Path())):
         """
         return self.mkdir(parents=True, exist_ok=True)
 
+    def with_suffix(self, suffix: str) -> 'Path':
+        """
+
+        Pathlib doesn't add a dot prefix, but then errors if you don't provide one, which feels rather obnoxious.
+
+        """
+        if not suffix.startswith('.'):
+            suffix = f'.{suffix}'
+        return super().with_suffix(suffix)
+
+    def get_conversion_path(self, suffix: str) -> 'Path':
+        """
+
+        Fetch the equivalent path for a different format in the standard conversion directory structure.
+        .../xyz/filename.xyx -> ../abc/filename.abc
+
+        """
+
+        old_dir = self.parent.name
+
+        if old_dir != self.suffix.removeprefix('.'):
+            raise ValueError(f"Expected parent directory '{old_dir}' to match file extension '{suffix}'")
+
+        new = self.parent.parent / suffix / f'{self.stem}.{suffix}'
+        return new
+
     @property
     def exist(self):
         """

fmtr/tools/pdf_tools.py

@@ -179,6 +179,20 @@ class Document(pm.Document):
         """
         return pymupdf4llm.to_markdown(self, **kwargs)
 
+    def to_text(self):
+        """
+
+        Simple text output.
+
+        """
+        lines = []
+        for page in self:
+            text = page.get_text()
+            lines.append(text)
+
+        text = '\n'.join(lines)
+        return text
+
 
 if __name__ == '__main__':
     from fmtr.tools.path_tools import Path

fmtr/tools/tabular_tools.py

@@ -1,7 +1,68 @@
+import deepdiff
+import numpy as np
 import pandas as pd
 
+from fmtr.tools.iterator_tools import dedupe
+
 Table = DataFrame = pd.DataFrame
 Series = pd.Series
 
+nan = np.nan
+
 CONCAT_HORIZONTALLY = 1
 CONCAT_VERTICALLY = 0
+
+
+def normalize_nan(df, value=np.nan):
+    return df.replace({pd.NA: value, None: value, np.nan: value})
+
+
+class Differ:
+    """
+
+    Diff two dataframes via DeepDiff, after shape normalization, datatype simplification, etc.
+
+    """
+
+    def __init__(self, left: Table, right: Table):
+
+        self.cols = dedupe(left.columns.tolist() + right.columns.tolist())
+        self.rows = dedupe(left.index.values.tolist() + right.index.values.tolist())
+        self.left = self.process(left)
+        self.right = self.process(right)
+        self.dfs = [self.left, self.right]
+
+    def process(self, df: Table) -> Table:
+        """
+
+        Ensure same rows/columns, plus simplify datatypes via JSON round-robin.
+
+        """
+
+        df_rows = set(df.index.values.tolist())
+        for row in self.rows:
+            if row in df_rows:
+                continue
+            df.loc[len(df)] = None
+
+        df_cols = set(df.columns.tolist())
+        for col in self.cols:
+            if col in df_cols:
+                continue
+            df[col] = None
+
+        df = pd.read_json(df.to_json(date_format='iso'))
+        df = normalize_nan(df, value=None)
+
+        return df
+
+    def get_diff(self) -> deepdiff.DeepDiff:
+        """
+
+        Cast to dicts and get diff
+
+        """
+
+        dicts = [df.to_dict(orient='index') for df in self.dfs]
+        diff = deepdiff.DeepDiff(*dicts, ignore_order=True)
+        return diff

fmtr/tools/version

@@ -1 +1 @@
-1.3.48
+1.3.49

{fmtr_tools-1.3.48.dist-info → fmtr_tools-1.3.49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fmtr.tools
-Version: 1.3.48
+Version: 1.3.49
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -81,16 +81,19 @@ Requires-Dist: pandas; extra == "semantic"
 Requires-Dist: tabulate; extra == "semantic"
 Requires-Dist: openpyxl; extra == "semantic"
 Requires-Dist: odfpy; extra == "semantic"
+Requires-Dist: deepdiff; extra == "semantic"
 Provides-Extra: metric
 Requires-Dist: pandas; extra == "metric"
 Requires-Dist: tabulate; extra == "metric"
 Requires-Dist: openpyxl; extra == "metric"
 Requires-Dist: odfpy; extra == "metric"
+Requires-Dist: deepdiff; extra == "metric"
 Provides-Extra: tabular
 Requires-Dist: pandas; extra == "tabular"
 Requires-Dist: tabulate; extra == "tabular"
 Requires-Dist: openpyxl; extra == "tabular"
 Requires-Dist: odfpy; extra == "tabular"
+Requires-Dist: deepdiff; extra == "tabular"
 Provides-Extra: html
 Requires-Dist: html2text; extra == "html"
 Provides-Extra: interface
@@ -151,67 +154,68 @@ Provides-Extra: db-document
 Requires-Dist: beanie[odm]; extra == "db-document"
 Requires-Dist: motor; extra == "db-document"
 Provides-Extra: all
-Requires-Dist: dnspython[doh]; extra == "all"
-Requires-Dist: playwright; extra == "all"
-Requires-Dist: yamlscript; extra == "all"
-Requires-Dist: uvicorn[standard]; extra == "all"
-Requires-Dist: logfire[fastapi]; extra == "all"
-Requires-Dist: beanie[odm]; extra == "all"
-Requires-Dist: cachetools; extra == "all"
-Requires-Dist: google-auth-oauthlib; extra == "all"
-Requires-Dist: deepmerge; extra == "all"
-Requires-Dist: appdirs; extra == "all"
-Requires-Dist: semver; extra == "all"
-Requires-Dist: contexttimer; extra == "all"
-Requires-Dist: Unidecode; extra == "all"
-Requires-Dist: pycountry; extra == "all"
-Requires-Dist: html2text; extra == "all"
-Requires-Dist: setuptools; extra == "all"
-Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
-Requires-Dist: bokeh; extra == "all"
-Requires-Dist: pydantic-extra-types; extra == "all"
-Requires-Dist: pymupdf; extra == "all"
-Requires-Dist: openpyxl; extra == "all"
-Requires-Dist: flet-video; extra == "all"
 Requires-Dist: pydantic-ai[logfire,openai]; extra == "all"
+Requires-Dist: torchaudio; extra == "all"
+Requires-Dist: odfpy; extra == "all"
+Requires-Dist: pymupdf; extra == "all"
 Requires-Dist: distributed; extra == "all"
 Requires-Dist: json_repair; extra == "all"
-Requires-Dist: dask[bag]; extra == "all"
-Requires-Dist: google-auth; extra == "all"
+Requires-Dist: bokeh; extra == "all"
 Requires-Dist: tinynetrc; extra == "all"
-Requires-Dist: diskcache; extra == "all"
 Requires-Dist: fastapi; extra == "all"
-Requires-Dist: httpx; extra == "all"
-Requires-Dist: logfire[httpx]; extra == "all"
-Requires-Dist: motor; extra == "all"
-Requires-Dist: ollama; extra == "all"
-Requires-Dist: flet[all]; extra == "all"
-Requires-Dist: regex; extra == "all"
+Requires-Dist: pycountry; extra == "all"
 Requires-Dist: sre_yield; extra == "all"
-Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: semver; extra == "all"
+Requires-Dist: html2text; extra == "all"
 Requires-Dist: pytest-cov; extra == "all"
-Requires-Dist: faker; extra == "all"
 Requires-Dist: pyyaml; extra == "all"
-Requires-Dist: torchaudio; extra == "all"
-Requires-Dist: sentence_transformers; extra == "all"
-Requires-Dist: tabulate; extra == "all"
-Requires-Dist: google-auth-httplib2; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: google-api-python-client; extra == "all"
+Requires-Dist: appdirs; extra == "all"
+Requires-Dist: pydantic; extra == "all"
 Requires-Dist: httpx_retries; extra == "all"
+Requires-Dist: ollama; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: pydantic-extra-types; extra == "all"
+Requires-Dist: google-auth; extra == "all"
+Requires-Dist: dask[bag]; extra == "all"
+Requires-Dist: uvicorn[standard]; extra == "all"
+Requires-Dist: httpx; extra == "all"
 Requires-Dist: pydantic-settings; extra == "all"
-Requires-Dist: peft; extra == "all"
+Requires-Dist: sentence_transformers; extra == "all"
+Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: deepdiff; extra == "all"
+Requires-Dist: diskcache; extra == "all"
+Requires-Dist: faker; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: contexttimer; extra == "all"
+Requires-Dist: google-api-python-client; extra == "all"
 Requires-Dist: filetype; extra == "all"
-Requires-Dist: torchvision; extra == "all"
-Requires-Dist: flet-webview; extra == "all"
-Requires-Dist: huggingface_hub; extra == "all"
-Requires-Dist: docker; extra == "all"
-Requires-Dist: odfpy; extra == "all"
-Requires-Dist: pydantic; extra == "all"
+Requires-Dist: motor; extra == "all"
+Requires-Dist: deepmerge; extra == "all"
+Requires-Dist: peft; extra == "all"
+Requires-Dist: yamlscript; extra == "all"
+Requires-Dist: transformers[sentencepiece]; extra == "all"
+Requires-Dist: google-auth-oauthlib; extra == "all"
 Requires-Dist: pymupdf4llm; extra == "all"
+Requires-Dist: setuptools; extra == "all"
+Requires-Dist: tabulate; extra == "all"
 Requires-Dist: logfire; extra == "all"
-Requires-Dist: tokenizers; extra == "all"
+Requires-Dist: beanie[odm]; extra == "all"
+Requires-Dist: pydevd-pycharm~=251.25410.159; extra == "all"
+Requires-Dist: flet-video; extra == "all"
+Requires-Dist: logfire[httpx]; extra == "all"
+Requires-Dist: regex; extra == "all"
 Requires-Dist: openai; extra == "all"
+Requires-Dist: docker; extra == "all"
+Requires-Dist: huggingface_hub; extra == "all"
+Requires-Dist: dnspython[doh]; extra == "all"
+Requires-Dist: google-auth-httplib2; extra == "all"
+Requires-Dist: Unidecode; extra == "all"
+Requires-Dist: logfire[fastapi]; extra == "all"
+Requires-Dist: openpyxl; extra == "all"
+Requires-Dist: torchvision; extra == "all"
+Requires-Dist: cachetools; extra == "all"
+Requires-Dist: flet-webview; extra == "all"
+Requires-Dist: flet[all]; extra == "all"
 Dynamic: author
 Dynamic: author-email
 Dynamic: description

{fmtr_tools-1.3.48.dist-info → fmtr_tools-1.3.49.dist-info}/RECORD

@@ -21,7 +21,7 @@ fmtr/tools/http_tools.py,sha256=RVwGrBNMyjfbpgAPCSnxEkXfSzXXWARb3ayq981ONQE,464
 fmtr/tools/import_tools.py,sha256=XJmiWLukRncJAcaGReDn4jIz1_IpVBjfYCQHH1hIg7c,588
 fmtr/tools/inherit_tools.py,sha256=gTGL4mRm5RsbFW76s25AbuAJ2vlymbh1c8Q4Hl2uJGU,646
 fmtr/tools/inspection_tools.py,sha256=tLTRvzy9XVomQPi0dfnF_cgwc7KiDVZAr7gPTk4S_bQ,278
-fmtr/tools/iterator_tools.py,sha256=ysNT2h39_ukEGrj8k7Z_CLKjWoguKBqVdjj4PLe7faE,1502
+fmtr/tools/iterator_tools.py,sha256=ymxo2U9MrPhouIhWCVvh1TrP1bXJPm_p0Lqwgi5Jr6w,1628
 fmtr/tools/json_fix_tools.py,sha256=vNSlswVQnujPmKEqDjFJcO901mjMyv59q3awsT7mlhs,477
 fmtr/tools/json_tools.py,sha256=WkFc5q7oqMtcFejhN1K5zQFULa9TdLOup83Fr0saDRY,348
 fmtr/tools/logging_tools.py,sha256=M7I5igs_tX5SIRv4f-jfb75LOODclSdmEg5ziAAMSPE,2503
@@ -33,7 +33,7 @@ fmtr/tools/openai_tools.py,sha256=6SUgejgzUzmlKKct2_ePXntvMegu3FJgfk9x7aqtqYc,74
 fmtr/tools/packaging_tools.py,sha256=FlgOTnDRHZWQL2iR-wucTsyGEHRE-MlddKL30MPmUqE,253
 fmtr/tools/parallel_tools.py,sha256=QEb_gN1StkxsqYaH4HSjiJX8Y3gpb2uKNsOzG4uFpaM,3071
 fmtr/tools/pattern_tools.py,sha256=DlEKzNJKhwFmU3-awoGkN5Xy-yLF_bsoj8eoSMCEytE,6018
-fmtr/tools/pdf_tools.py,sha256=xvv9B84uAF81rFJRnXhSsxYuP42vY9ZdPVFrSMVe8G8,4069
+fmtr/tools/pdf_tools.py,sha256=9ElZRvXtYlvZkw8koB31xSGDM2B9JHQT7FVVIaLoXQI,4311
 fmtr/tools/platform_tools.py,sha256=7p69CmAHe_sF68Fx9uVhns1k5EewTHTWgUYzkl6ZQKA,308
 fmtr/tools/process_tools.py,sha256=Ysh5Dk2QFBhXQerArjKdt7xZd3JrN5Ho02AaOjH0Nnw,1425
 fmtr/tools/profiling_tools.py,sha256=jpXVjaNKPydTasEQVNXvxzGtMhXPit08AnJddkU8uIc,46
@@ -42,11 +42,11 @@ fmtr/tools/semantic_tools.py,sha256=cxY9NSAHWj4nEc6Oj4qA1omR3dWbl2OuH7_PkINc6_E,
 fmtr/tools/settings_tools.py,sha256=o11W3T60UZSvCTkh_eEQq1Mx74GycQ6JxUr0plBDbsk,2356
 fmtr/tools/spaces_tools.py,sha256=D_he3mve6DruB3OPS6QyzqD05ChHnRTb4buViKPe7To,1099
 fmtr/tools/string_tools.py,sha256=On6YRLTAK1i6mmMpOUWVM618CykJiuaoyKIsU1cB_mA,4952
-fmtr/tools/tabular_tools.py,sha256=tpIpZzYku1HcJrHZJL6BC39LmN3WUWVhFbK2N7nDVmE,120
+fmtr/tools/tabular_tools.py,sha256=mw6vOij1Ch-pVAyHMPtm5zj__ULZN_TKeBYOfj33wFM,1634
 fmtr/tools/tokenization_tools.py,sha256=me-IBzSLyNYejLybwjO9CNB6Mj2NYfKPaOVThXyaGNg,4268
 fmtr/tools/tools.py,sha256=CAsApa1YwVdNE6H66Vjivs_mXYvOas3rh7fPELAnTpk,795
 fmtr/tools/unicode_tools.py,sha256=yS_9wpu8ogNoiIL7s1G_8bETFFO_YQlo4LNPv1NLDeY,52
-fmtr/tools/version,sha256=kV64L6ux0lCm-Ikwzdf6Z3DVqpQjGnzt_RRQ-TAwSug,6
+fmtr/tools/version,sha256=Gu0FxrnxSfSiaBEXNfjNzhDJ5oAjWBiF3qy9XGVZUJU,6
 fmtr/tools/webhook_tools.py,sha256=q3pVJ1NCem2SrMuFcLxiWd7DibFs7Q-uGtojfXd3Qcg,380
 fmtr/tools/yaml_tools.py,sha256=Bhhyd6GQVKO72Lp8ky7bAUjIB_65Hdh0Q45SKIEe6S8,1901
 fmtr/tools/ai_tools/__init__.py,sha256=O8VRlPnnQCncg2ZZ2l_VdWLJf4jkKH6dkZFVbv6o7IM,388
@@ -71,7 +71,7 @@ fmtr/tools/interface_tools/controls.py,sha256=oOl0_sZB8fkvYB-9A5yjArfQmFQLMCsVGg
 fmtr/tools/interface_tools/interface_tools.py,sha256=i1TqP_67pVGiRZotKdVxyH0b5OFwaTZ_5Tf60gip0ts,4214
 fmtr/tools/path_tools/__init__.py,sha256=XrJXt7Zzo90tYUVksMlDfKkWt775zJ9OSi2NbhnqMDI,459
 fmtr/tools/path_tools/app_path_tools.py,sha256=JrJvtTDd_gkCKcZtBCDTMktsM77PZwGV_hzQX0g5GU8,1722
-fmtr/tools/path_tools/path_tools.py,sha256=eh30PpmH0wopy0wNWuPT84cmXY1EvqsTSDT7AV_GPOY,8034
+fmtr/tools/path_tools/path_tools.py,sha256=s3RTXsjnr2Ah7vXQGjpGs-4DlWaVGvChu0aTFXX3gsE,8867
 fmtr/tools/path_tools/type_path_tools.py,sha256=Zgs-ek-GXRKDIlVDGdg3muB0PIxTg2ba0NeHw6y8FWQ,40
 fmtr/tools/setup_tools/__init__.py,sha256=Ro_Qj3Xndv8Z68DeWPI7c6X-aWKsdDm0KcX_k1xDhgE,394
 fmtr/tools/setup_tools/setup_tools.py,sha256=7Z6jlU6UE8P4cntGQ_hJR7hGvoqwh15xzZY63cnxG7E,10363
@@ -85,9 +85,9 @@ fmtr/tools/tests/test_path.py,sha256=AkZQa6_8BQ-VaCyL_J-iKmdf2ZaM-xFYR37Kun3k4_g
 fmtr/tools/tests/test_yaml.py,sha256=jc0TwwKu9eC0LvFGNMERdgBue591xwLxYXFbtsRwXVM,287
 fmtr/tools/version_tools/__init__.py,sha256=cjE6nO6AoVOUp3RwgTbqL9wiw8J1l2pHJOz6Gn6bxjA,326
 fmtr/tools/version_tools/version_tools.py,sha256=Hcc6yferZS1hHbugRTdiHhSNmXEEG0hjCiTTXKna-YY,1127
-fmtr_tools-1.3.48.dist-info/licenses/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
-fmtr_tools-1.3.48.dist-info/METADATA,sha256=sSq6t3Hgu4AHxBdNAj4TKme9fmZsbJdI-HQxSLeeb_0,17257
-fmtr_tools-1.3.48.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fmtr_tools-1.3.48.dist-info/entry_points.txt,sha256=h-r__Xh5njtFqreMLg6cGuTFS4Qh-QqJPU1HB-_BS-Q,357
-fmtr_tools-1.3.48.dist-info/top_level.txt,sha256=LXem9xCgNOD72tE2gRKESdiQTL902mfFkwWb6-dlwEE,5
-fmtr_tools-1.3.48.dist-info/RECORD,,
+fmtr_tools-1.3.49.dist-info/licenses/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
+fmtr_tools-1.3.49.dist-info/METADATA,sha256=1oejOVboa3Ol_fXNcxIa3WG49h9fMV8G8YgRWGqeJJo,17429
+fmtr_tools-1.3.49.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fmtr_tools-1.3.49.dist-info/entry_points.txt,sha256=h-r__Xh5njtFqreMLg6cGuTFS4Qh-QqJPU1HB-_BS-Q,357
+fmtr_tools-1.3.49.dist-info/top_level.txt,sha256=LXem9xCgNOD72tE2gRKESdiQTL902mfFkwWb6-dlwEE,5
+fmtr_tools-1.3.49.dist-info/RECORD,,