fmtr.tools 1.0.31__py3-none-any.whl → 1.0.33__py3-none-any.whl

fmtr/tools/__init__.py

@@ -143,6 +143,11 @@ try:
 except ImportError as exception:
     caching = MissingExtraMockModule('caching', exception)
 
+try:
+    from fmtr.tools import pdf_tools as pdf
+except ImportError as exception:
+    pdf = MissingExtraMockModule('pdf', exception)
+
 
 __all__ = [
     'config',

fmtr/tools/api_tools.py

@@ -83,6 +83,11 @@ class ApiBase:
 
     @classmethod
     def launch(cls):
+        """
+
+        Initialise self and launch.
+
+        """
         self = cls()
         logger.info(f'Launching API {cls.TITLE}...')
         uvicorn.run(self.app, host=self.HOST, port=self.PORT)

fmtr/tools/path_tools.py

@@ -96,6 +96,29 @@ class Path(type(Path())):
         """
         return cls(gettempdir())
 
+    @classmethod
+    def data(cls, name='data') -> 'Path':
+        """
+
+        Fetch canonical "data"/"artifacts" path, whether calling package is regular or namespace package.
+
+        """
+        from fmtr.tools.inspection_tools import get_call_path
+        path = get_call_path()
+        path = path.absolute().parent.parent
+
+        path /= name
+
+        if path.exists():
+            return path
+
+        path = path.parent.parent / name
+
+        if path.exists():
+            return path
+
+        raise FileNotFoundError(f'No "{name}" directory found at "{path}"')
+
     def write_json(self, obj) -> int:
         """
 

fmtr/tools/pdf_tools.py

@@ -0,0 +1,172 @@
+import pymupdf as pm
+import pymupdf4llm
+from typing import List, Tuple, Dict, Any, Self
+
+from fmtr.tools import data_modelling_tools
+
+
+class BoundingBox(data_modelling_tools.Base):
+    left: float
+    top: float
+    right: float
+    bottom: float
+
+    @property
+    def order(self):
+        """
+
+        Approximate natural reading order
+
+        """
+        return (self.top, self.left), (self.bottom, self.right)
+
+    @property
+    def rect(self) -> pm.Rect:
+        """
+
+        Position as a PyMuPDF Rect
+
+        """
+        return pm.Rect(self.left, self.top, self.right, self.bottom)
+
+    @classmethod
+    def from_dict(cls, data: Tuple[float]) -> Self:
+        """
+
+        Instantiate from PyMuPDF dictionary data
+
+        """
+        data = {key: value for key, value in zip(cls.model_fields.keys(), data)}
+        return cls(**data)
+
+
+class Span(data_modelling_tools.Base):
+    size: float
+    flags: int
+    font: str
+    color: int
+    ascender: float
+    descender: float
+    text: str
+    origin: Tuple[float, float]
+    bbox: BoundingBox
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> Self:
+        data['bbox'] = BoundingBox.from_dict(data['bbox'])
+        return cls(**data)
+
+
+class Line(data_modelling_tools.Base):
+    spans: List[Span]
+    wmode: int
+    dir: Tuple[float, float]
+    bbox: BoundingBox
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'Line':
+        """
+
+        Instantiate from PyMuPDF dictionary data
+
+        """
+        data['spans'] = [Span.from_dict(span) for span in data['spans']]
+        data['bbox'] = BoundingBox.from_dict(data['bbox'])
+        return cls(**data)
+
+    @property
+    def text(self) -> str:
+        """
+
+        Simple text representation
+
+        """
+        return ' '.join([span.text for span in self.spans])
+
+
+class Block(data_modelling_tools.Base):
+    number: int
+    type: int
+    bbox: BoundingBox
+    lines: List[Line]
+
+    @property
+    def text(self) -> str:
+        """
+
+        Simple text representation
+
+        """
+        return ' '.join([span.text for span in self.spans])
+
+    @classmethod
+    def from_dict(cls, data: Dict) -> Self:
+        """
+
+        Instantiate from PyMuPDF dictionary data
+
+        """
+        data['lines'] = [Line.from_dict(line) for line in data['lines']]
+        data['bbox'] = BoundingBox.from_dict(data['bbox'])
+        return cls(**data)
+
+    @property
+    def rect(self) -> pm.Rect:
+        """
+
+        Position as a PyMuPDF Rect
+
+        """
+        return self.bbox.rect
+
+
+class Document(pm.Document):
+    """
+
+    Subclassed Document object with data-modelled elements property and markdown conversion.
+
+    """
+
+    @property
+    def data(self) -> List[Block]:
+        """
+
+        Get representation of Document elements as Python objects.
+
+        """
+
+        blocks = []
+
+        for page in self:
+            for block in page.get_text("dict")["blocks"]:
+                obj = Block.from_dict(block)
+                blocks.append(obj)
+
+        return blocks
+
+    def to_markdown(self, **kwargs) -> str:
+        """
+
+        Markdown output via `pymupdf4llm`
+
+        """
+        return pymupdf4llm.to_markdown(self, **kwargs)
+
+
+if __name__ == '__main__':
+    from fmtr.tools.path_tools import Path
+
+    PATH_DATA = Path.data()
+    # PATH_PDF=PATH_DATA/'chib.pdf'
+    PATH_PDF = PATH_DATA / 'kvm.pdf'
+    assert PATH_PDF.exists()
+
+    doc = Document(PATH_PDF)
+    doc.data
+
+    for page in doc:
+        print(page.get_text('dict'))
+        print(page.get_text('html'))
+
+    md = doc.to_markdown()
+    md

fmtr/tools/string_tools.py

@@ -145,3 +145,10 @@ class Mask:
             return text
         except (KeyError, IndexError):
             return self
+
+
+if __name__ == '__main__':
+    import numpy as np
+
+    st = join([1, None, 'test', np.nan, 0, '', 'yeah'])
+    st

fmtr/tools/version

@@ -1 +1 @@
-1.0.31
+1.0.33

{fmtr.tools-1.0.31.dist-info → fmtr.tools-1.0.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fmtr.tools
-Version: 1.0.31
+Version: 1.0.33
 Summary: Collection of high-level tools to simplify everyday development tasks, with a focus on AI/ML
 Home-page: https://github.com/fmtr/fmtr.tools
 Author: Frontmatter
@@ -67,6 +67,10 @@ Provides-Extra: parallel
 Requires-Dist: bokeh ; extra == 'parallel'
 Requires-Dist: dask[bag] ; extra == 'parallel'
 Requires-Dist: distributed ; extra == 'parallel'
+Provides-Extra: pdf
+Requires-Dist: pydantic ; extra == 'pdf'
+Requires-Dist: pymupdf ; extra == 'pdf'
+Requires-Dist: pymupdf4llm ; extra == 'pdf'
 Provides-Extra: process
 Requires-Dist: logfire ; extra == 'process'
 Requires-Dist: semver ; extra == 'process'
@@ -105,6 +109,8 @@ Requires-Dist: pandas ; extra == 'test'
 Requires-Dist: peft ; extra == 'test'
 Requires-Dist: pydantic ; extra == 'test'
 Requires-Dist: pydantic-ai[logfire,openai] ; extra == 'test'
+Requires-Dist: pymupdf ; extra == 'test'
+Requires-Dist: pymupdf4llm ; extra == 'test'
 Requires-Dist: pytest-cov ; extra == 'test'
 Requires-Dist: pyyaml ; extra == 'test'
 Requires-Dist: semver ; extra == 'test'

{fmtr.tools-1.0.31.dist-info → fmtr.tools-1.0.33.dist-info}/RECORD

@@ -1,6 +1,6 @@
-fmtr/tools/__init__.py,sha256=imKjjZg5HypnRy5bD4Xe6HBkQhavg6BOkwIeFzItG_8,5052
+fmtr/tools/__init__.py,sha256=3eJEUYhp82YvQUmoaFdNYaiNeQnyoRuxmez0uqKgZdU,5186
 fmtr/tools/ai_tools.py,sha256=hN7DzuATXfurCDHugaluUsbmF_PzeKu3BTc2WXhG59g,11806
-fmtr/tools/api_tools.py,sha256=KU_qetI7WQRZijZ0l1nCKjPcRbseMMCWj0W8n_NTJCM,2027
+fmtr/tools/api_tools.py,sha256=u5YEdKyKto8MKY8legULLU7xeJ7lY2Bgyaep_xa8iZg,2089
 fmtr/tools/async_tools.py,sha256=ewz757WcveQJd-G5SVr2JDOQVbdLGecCgl-tsBGVZz4,284
 fmtr/tools/augmentation_tools.py,sha256=-6ESbO4CDlKqVOV1J1V6qBeoBMzbFIinkDHRHnCBej0,55
 fmtr/tools/caching_tools.py,sha256=UOCYUNvLQ-NofR_dhqBmZF96-HRPf4At5MmxVk3gAIk,2943
@@ -30,18 +30,19 @@ fmtr/tools/name_tools.py,sha256=5CB_phqhHjl66iI8oLxOGPF2odC1apdul-M8Fv2xBhs,5514
 fmtr/tools/netrc_tools.py,sha256=PpNpz_mWlQi6VHGromKwFfTyLpHUXsd4LY6-OKLCbeI,376
 fmtr/tools/openai_tools.py,sha256=6SUgejgzUzmlKKct2_ePXntvMegu3FJgfk9x7aqtqYc,742
 fmtr/tools/parallel_tools.py,sha256=G__ZbLRRx4cP5OyqY1hKwnE-VI3m5prYABB0tnZHnes,3132
-fmtr/tools/path_tools.py,sha256=iNIG-nxBX-uiyfyOapJZaUiS4pI8sHsxm6WhVHixQ0M,3700
+fmtr/tools/path_tools.py,sha256=1GeWXdhV5rH99IfLI5ZFEnOJfs4Q4mYTT2R-rA791iQ,4273
+fmtr/tools/pdf_tools.py,sha256=3XZ6Tpvuf46oeYENsYQsYn-6hbTqHDSJesqNZeJ1Rko,3563
 fmtr/tools/platform_tools.py,sha256=7p69CmAHe_sF68Fx9uVhns1k5EewTHTWgUYzkl6ZQKA,308
 fmtr/tools/process_tools.py,sha256=Ysh5Dk2QFBhXQerArjKdt7xZd3JrN5Ho02AaOjH0Nnw,1425
 fmtr/tools/profiling_tools.py,sha256=jpXVjaNKPydTasEQVNXvxzGtMhXPit08AnJddkU8uIc,46
 fmtr/tools/random_tools.py,sha256=4VlQdk5THbR8ka4pZaLbk_ZO_4yy6PF_lHZes_rgenY,2223
 fmtr/tools/semantic_tools.py,sha256=cxY9NSAHWj4nEc6Oj4qA1omR3dWbl2OuH7_PkINc6_E,1386
 fmtr/tools/spaces_tools.py,sha256=D_he3mve6DruB3OPS6QyzqD05ChHnRTb4buViKPe7To,1099
-fmtr/tools/string_tools.py,sha256=w0lw70bgzJ8tAHj_4lMrjtMyefE5kELgpCBgGzGcalo,3117
+fmtr/tools/string_tools.py,sha256=U2EptMWR6KDOP22ZQ4ReUHV4i25SP7xwCmZScI1sy4M,3233
 fmtr/tools/tokenization_tools.py,sha256=9FP5vgPufWv0XA961eVKObFll0d_2mM0W3ut3rtZyeo,4329
 fmtr/tools/tools.py,sha256=xnfUrOnrT4OxFYez6vV5tAhydzCICJFiGVnviiZDEQo,796
 fmtr/tools/unicode_tools.py,sha256=yS_9wpu8ogNoiIL7s1G_8bETFFO_YQlo4LNPv1NLDeY,52
-fmtr/tools/version,sha256=VmLTmBbOULW3noulizLqu_B8fqxN0ySSfEW-8f3y1Sk,6
+fmtr/tools/version,sha256=V-CY1GAptD08dwDUv8tP0IXGjOPKFQq8iTv5XUR7qhc,6
 fmtr/tools/version_tools.py,sha256=axzzHBS9V1n6YuSacsDKG3VfAvRqR8qr6aENCibR8vs,1248
 fmtr/tools/yaml_tools.py,sha256=Ol43ZwbnSXGnn1K98Uxx61KPGSqfC4axE-X2q1LKMwk,349
 fmtr/tools/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -52,9 +53,9 @@ fmtr/tools/tests/test_environment.py,sha256=iHaiMQfECYZPkPKwfuIZV9uHuWe3aE-p_dN_
 fmtr/tools/tests/test_json.py,sha256=IeSP4ziPvRcmS8kq7k9tHonC9rN5YYq9GSNT2ul6Msk,287
 fmtr/tools/tests/test_path.py,sha256=AkZQa6_8BQ-VaCyL_J-iKmdf2ZaM-xFYR37Kun3k4_g,2188
 fmtr/tools/tests/test_yaml.py,sha256=jc0TwwKu9eC0LvFGNMERdgBue591xwLxYXFbtsRwXVM,287
-fmtr.tools-1.0.31.dist-info/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
-fmtr.tools-1.0.31.dist-info/METADATA,sha256=VOhy728mlqRmzO9jP8wUEN8rWHYap9dcf9AA0beu9r8,13207
-fmtr.tools-1.0.31.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-fmtr.tools-1.0.31.dist-info/entry_points.txt,sha256=CEStVkwJ1mTFvhN1WV5RdW83SkNW1d5Syj-KZ6A19ng,72
-fmtr.tools-1.0.31.dist-info/top_level.txt,sha256=t5341a8ii3n4RFizwTeXGmcq_pf4GqL1h9ylE5LIWRk,12
-fmtr.tools-1.0.31.dist-info/RECORD,,
+fmtr.tools-1.0.33.dist-info/LICENSE,sha256=FW9aa6vVN5IjRQWLT43hs4_koYSmpcbIovlKeAJ0_cI,10757
+fmtr.tools-1.0.33.dist-info/METADATA,sha256=G6ybzpAyBJfpfyoTPLQBXB6HJRrG-3lKwc84YaSI9nI,13438
+fmtr.tools-1.0.33.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+fmtr.tools-1.0.33.dist-info/entry_points.txt,sha256=CEStVkwJ1mTFvhN1WV5RdW83SkNW1d5Syj-KZ6A19ng,72
+fmtr.tools-1.0.33.dist-info/top_level.txt,sha256=t5341a8ii3n4RFizwTeXGmcq_pf4GqL1h9ylE5LIWRk,12
+fmtr.tools-1.0.33.dist-info/RECORD,,