extract-python 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

extract_python/docling_.py

@@ -32,6 +32,7 @@ from .objects import (
     PageIndexes,
     Result,
     Status,
+    SupportedExt,
 )
 from .pipeline import Pipeline, PipelineConfig, PipelineType
 from .utils import all_subclasses, chdir, map_and_preserve, path_to_artifacts_dirname
@@ -136,6 +137,30 @@ class DoclingPipelineConfig(PipelineConfig):
             for f, opt in self.format_options.items()
         }
 
+    @classmethod
+    @cache
+    def supported_formats(cls) -> set[SupportedExt]:
+        # Subset of https://docling-project.github.io/docling/usage/supported_formats/
+        return {
+            SupportedExt.ADOC,
+            SupportedExt.ASCIIDOC,
+            SupportedExt.BMP,
+            SupportedExt.CSV,
+            SupportedExt.DOCX,
+            SupportedExt.HTLM,
+            SupportedExt.JPG,
+            SupportedExt.MD,
+            SupportedExt.PDF,
+            SupportedExt.PNG,
+            SupportedExt.PPTX,
+            SupportedExt.TEX,
+            SupportedExt.TIFF,
+            SupportedExt.TXT,
+            SupportedExt.WEBP,
+            SupportedExt.XHTML,
+            SupportedExt.XLSX,
+        }
+
 
 DEFAULT_FORMAT_OPTIONS = DoclingPipelineConfig().to_format_options()
 

extract_python/marker_.py

@@ -1,6 +1,7 @@
 import gc
 from collections.abc import AsyncGenerator, Iterable
 from copy import deepcopy
+from functools import cache
 from pathlib import Path
 from typing import Any, ClassVar, Self
 
@@ -20,6 +21,7 @@ from .objects import (
     PageIndexes,
     Result,
     Status,
+    SupportedExt,
 )
 from .pipeline import Pipeline, PipelineConfig, PipelineType
 from .utils import path_to_artifacts_dirname, report_recoverable_errors
@@ -32,6 +34,33 @@ class MarkerPipelineConfig(PipelineConfig):
 
     config: dict[str, Any] = dict()
 
+    @classmethod
+    @cache
+    def supported_formats(cls) -> set[SupportedExt]:
+        # Subset of https://documentation.datalab.to/docs/common/supportedfiletypes
+        return {
+            SupportedExt.PDF,
+            SupportedExt.XLS,
+            SupportedExt.XLSX,
+            SupportedExt.XLSM,
+            SupportedExt.CSV,
+            SupportedExt.ODS,
+            SupportedExt.DOC,
+            SupportedExt.DOCX,
+            SupportedExt.ODT,
+            SupportedExt.PPT,
+            SupportedExt.PPTX,
+            SupportedExt.ODP,
+            SupportedExt.HTLM,
+            SupportedExt.EPUB,
+            SupportedExt.PNG,
+            SupportedExt.JPG,
+            SupportedExt.JPEG,
+            SupportedExt.WEBP,
+            SupportedExt.GIF,
+            SupportedExt.TIFF,
+        }
+
 
 _MARKER_CONVERSION_ERRORS = tuple()
 

extract_python/miner_u.py

@@ -3,7 +3,7 @@ import shutil
 from collections.abc import AsyncGenerator, Callable, Iterable
 from copy import copy
 from enum import StrEnum
-from functools import partial
+from functools import cache, partial
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, ClassVar, Self
@@ -26,6 +26,7 @@ from .objects import (
     PageIndexes,
     Result,
     Status,
+    SupportedExt,
 )
 from .pipeline import Pipeline, PipelineConfig, PipelineType
 from .utils import path_to_artifacts_dirname
@@ -82,6 +83,16 @@ class MinerUPipelineConfig(PipelineConfig):  # noqa: F821
     config: MinerUConfig = Field(frozen=True, default=MinerUConfig())
     language: LanguageAlpha2 = Field(frozen=True, default="en")
 
+    @classmethod
+    @cache
+    def supported_formats(cls) -> set[SupportedExt]:
+        return {
+            SupportedExt.PDF,
+            SupportedExt.DOCX,
+            SupportedExt.PPTX,
+            SupportedExt.XLSX,
+        }
+
 
 @Pipeline.register(PipelineType.MINER_U)
 class MinerUPipeline(Pipeline):

extract_python/objects.py

@@ -38,7 +38,34 @@ class BaseModel(_BaseModel):
 
 
 class SupportedExt(StrEnum):
+    ADOC = ".adoc"
+    ASCIIDOC = ".asciidoc"
+    BMP = ".bmp"
+    CSV = ".csv"
+    DOC = ".doc"
+    DOCX = ".docx"
+    EPUB = ".epub"
+    GIF = ".gif"
+    HTLM = ".html"
+    JPEG = ".jpeg"
+    JPG = ".jpg"
+    MD = ".md"
+    ODP = ".odp"
+    ODS = ".ods"
+    ODT = ".odt"
     PDF = ".pdf"
+    PNG = ".png"
+    PPT = ".ppt"
+    PPTX = ".pptx"
+    TEX = ".tex"
+    TIFF = ".tiff"
+    TXT = ".txt"
+    WEBP = ".webp"
+    XHTML = ".xhtml"
+    XLS = ".xls"
+    XLSM = ".xlsm"
+    XLSX = ".xlsx"
+    XLTX = ".xltx"
 
     def to_docling(self) -> InputFormat:
         return InputFormat(self.value[1:])

extract_python/pipeline.py

@@ -8,7 +8,7 @@ from icij_common.pydantic_utils import icij_config, merge_configs, no_enum_value
 from icij_common.registrable import RegistrableConfig, RegistrableFromConfig
 from pydantic import Field
 
-from .objects import InputDoc, OutputFormat, Result
+from .objects import InputDoc, OutputFormat, Result, SupportedExt
 
 StructuredContent = str
 
@@ -28,6 +28,10 @@ class PipelineConfig(RegistrableConfig, ABC):
 
     task_group: ClassVar[str] = Field(frozen=True)
 
+    @classmethod
+    @abstractmethod
+    def supported_formats(cls) -> set[SupportedExt]: ...
+
 
 class Pipeline(RegistrableFromConfig, ABC):
     @abstractmethod

{extract_python-0.1.0.dist-info → extract_python-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-python
-Version: 0.1.0
+Version: 0.2.1
 Summary: Structured content extraction
 Project-URL: Homepage, https://github.com/ICIJ/extract-python
 Project-URL: Repository, https://github.com/ICIJ/extract-python

extract_python-0.2.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+extract_python/__init__.py,sha256=Y-lcFbJd5sX3wt1UnyYAbm7H81Sgg7p2fG07CuuJUco,945
+extract_python/constants.py,sha256=JmAkjXyQMYwpTod9DCLc11zOc3caK2j_2ji_r3hGZws,236
+extract_python/docling_.py,sha256=L7RPOYTN4kuodsgfi7NVsXILRerbAZUDaYxqVofxOog,9267
+extract_python/marker_.py,sha256=EUGpBRAe9mE0QbSdMFdvE16_m-c-DeAtwZ8F79w2Mcg,4908
+extract_python/miner_u.py,sha256=Bse66I5Yj-PiOgejr3JjXXDkjCh46M9KuwTEB8QK5g4,7750
+extract_python/objects.py,sha256=hqa9ONk9KwylvQa6DvKIEQnnCgfy-T-d5SU2LpfmTcQ,7815
+extract_python/pipeline.py,sha256=0qkuqEcxEbc3_sy8gNbLPwq8IIlC8cfGaqk_5fNpOCM,1207
+extract_python/utils.py,sha256=kXe0CyT6zGYDTOvxs3BJYxq2-cgJZ0d_IT3hdMbuXa8,1943
+extract_python-0.2.1.dist-info/METADATA,sha256=iaIXzaha4s-kqJzkedwJn5VsvQGZhy3KJQJpDns8pR4,1132
+extract_python-0.2.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+extract_python-0.2.1.dist-info/RECORD,,

extract_python-0.1.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-extract_python/__init__.py,sha256=Y-lcFbJd5sX3wt1UnyYAbm7H81Sgg7p2fG07CuuJUco,945
-extract_python/constants.py,sha256=JmAkjXyQMYwpTod9DCLc11zOc3caK2j_2ji_r3hGZws,236
-extract_python/docling_.py,sha256=FMDsPVz05sGMPvIOX90lOLygWp6nC5DEjRfgx_ESPJ4,8530
-extract_python/marker_.py,sha256=z3PkUUStC-E78HhqByMwJ7re6-I7YUQzSxWToegHrUQ,4060
-extract_python/miner_u.py,sha256=f5pvLvay1ThBXNOI1R276aWSWsk5mhIPzWVjCy2u_lw,7493
-extract_python/objects.py,sha256=gTyGA5gaMAmW5P_PbAO2LNMqtP69CxlknebBFTojiwQ,7322
-extract_python/pipeline.py,sha256=qUgGar1rlYQgNz78BcUT1nQRsG3hy5UwpCl0e-0V77I,1098
-extract_python/utils.py,sha256=kXe0CyT6zGYDTOvxs3BJYxq2-cgJZ0d_IT3hdMbuXa8,1943
-extract_python-0.1.0.dist-info/METADATA,sha256=wyYMrleKk9yUU1UaTYT0EsGpw_e3qbE8LOBanyLv0Qg,1132
-extract_python-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-extract_python-0.1.0.dist-info/RECORD,,