docling-core 1.7.2__py3-none-any.whl → 2.0.0__py3-none-any.whl

docling_core/types/rec/subject.py

@@ -15,7 +15,7 @@ from docling_core.types.base import (
     SubjectNameTypeT,
     SubjectTypeT,
 )
-from docling_core.types.doc.base import S3Reference
+from docling_core.types.legacy_doc.base import S3Reference
 from docling_core.utils.alias import AliasModel
 
 

docling_core/utils/generate_docs.py

@@ -0,0 +1,82 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Generate documentation of Docling types as JSON schema.
+
+Example:
+    python docling_core/utils/generate_docs.py /tmp/docling_core_files
+"""
+import argparse
+import json
+import os
+from argparse import BooleanOptionalAction
+from pathlib import Path
+from shutil import rmtree
+from typing import Final
+
+from docling_core.utils.generate_jsonschema import generate_json_schema
+
+MODELS: Final = ["Document", "Record", "Generic"]
+
+
+def _prepare_directory(folder: str, clean: bool = False) -> None:
+    """Create a directory or empty its content if it already exists.
+
+    Args:
+        folder: The name of the directory.
+        clean: Whether any existing content in the directory should be removed.
+    """
+    if os.path.isdir(folder):
+        if clean:
+            for path in Path(folder).glob("**/*"):
+                if path.is_file():
+                    path.unlink()
+                elif path.is_dir():
+                    rmtree(path)
+    else:
+        os.makedirs(folder, exist_ok=True)
+
+
+def generate_collection_jsonschema(folder: str):
+    """Generate the JSON schema of Docling collections and export them to a folder.
+
+    Args:
+        folder: The name of the directory.
+    """
+    for item in MODELS:
+        json_schema = generate_json_schema(item)
+        with open(
+            os.path.join(folder, f"{item}.json"), mode="w", encoding="utf8"
+        ) as json_file:
+            json.dump(json_schema, json_file, ensure_ascii=False, indent=2)
+
+
+def main() -> None:
+    """Generate the JSON Schema of Docling collections and export documentation."""
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        "directory",
+        help=(
+            "Directory to generate files. If it exists, any existing content will be"
+            " removed."
+        ),
+    )
+    argparser.add_argument(
+        "--clean",
+        help="Whether any existing content in directory should be removed.",
+        action=BooleanOptionalAction,
+        dest="clean",
+        default=False,
+        required=False,
+    )
+    args = argparser.parse_args()
+
+    _prepare_directory(args.directory, args.clean)
+
+    generate_collection_jsonschema(args.directory)
+
+
+if __name__ == "__main__":
+    main()

docling_core/utils/{ds_generate_jsonschema.py → generate_jsonschema.py}

@@ -6,7 +6,7 @@
 """Generate the JSON Schema of pydantic models and export them to files.
 
 Example:
-    python docling_core/utils/ds_generate_jsonschema.py doc.base.TableCell
+    python docling_core/utils/generate_jsonschema.py doc.document.TableCell
 
 """
 import argparse
@@ -27,10 +27,10 @@ def _import_class(class_reference: str) -> Any:
 
 
 def generate_json_schema(class_reference: str) -> Union[dict, None]:
-    """Generate a jsonable dict of a model's schema from DS data types.
+    """Generate a jsonable dict of a model's schema from a data type.
 
     Args:
-        class_reference: The reference to a class in 'src.data_types'.
+        class_reference: The reference to a class in 'docling_core.types'.
 
     Returns:
         A jsonable dict of the model's schema.
@@ -48,7 +48,7 @@ def main() -> None:
     """Print the JSON Schema of a model."""
     argparser = argparse.ArgumentParser()
     argparser.add_argument(
-        "class_ref", help="Class reference, e.g., doc.base.TableCell"
+        "class_ref", help="Class reference, e.g., doc.document.TableCell"
     )
     args = argparser.parse_args()
 

docling_core/utils/validators.py

@@ -38,7 +38,7 @@ def validate_raw_schema(file_: dict) -> tuple[bool, str]:
 
     schema_txt = (
         resources.files("docling_core")
-        .joinpath("resources/schemas/doc/RAW.json")
+        .joinpath("resources/schemas/legacy_doc/RAW.json")
         .read_text("utf-8")
     )
     schema = json.loads(schema_txt)
@@ -52,7 +52,7 @@ def validate_ann_schema(file_: dict) -> tuple[bool, str]:
 
     schema_txt = (
         resources.files("docling_core")
-        .joinpath("resources/schemas/doc/ANN.json")
+        .joinpath("resources/schemas/legacy_doc/ANN.json")
         .read_text("utf-8")
     )
     schema = json.loads(schema_txt)
@@ -66,7 +66,7 @@ def validate_ocr_schema(file_: dict) -> tuple[bool, str]:
 
     schema_txt = (
         resources.files("docling_core")
-        .joinpath("resources/schemas/doc/OCR-output.json")
+        .joinpath("resources/schemas/legacy_doc/OCR-output.json")
         .read_text("utf-8")
     )
     schema = json.loads(schema_txt)

{docling_core-1.7.2.dist-info → docling_core-2.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling-core
-Version: 1.7.2
+Version: 2.0.0
 Summary: A python library to define and validate data types in Docling.
 Home-page: https://ds4sd.github.io/
 License: MIT
@@ -25,10 +25,10 @@ Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
-Requires-Dist: json-schema-for-humans (>=1.0.0,<2.0.0)
 Requires-Dist: jsonref (>=1.1.0,<2.0.0)
 Requires-Dist: jsonschema (>=4.16.0,<5.0.0)
 Requires-Dist: pandas (>=2.1.4,<3.0.0)
+Requires-Dist: pillow (>=10.3.0,<11.0.0)
 Requires-Dist: pydantic (>=2.6.0,<3.0.0)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Project-URL: Repository, https://github.com/DS4SD/docling-core
@@ -37,7 +37,7 @@ Description-Content-Type: text/markdown
 # Docling Core
 
 [![PyPI version](https://img.shields.io/pypi/v/docling-core)](https://pypi.org/project/docling-core/)
-![Python](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue)
+![Python](https://img.shields.io/badge/python-3.9%20%7C%203.10%20%7C%20%203.11%20%7C%203.12%20%7C%203.13-blue)
 [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)
@@ -57,7 +57,7 @@ pip install docling-core
 
 ### Development setup
 
-To develop for Docling Core, you need Python 3.9 / 3.10 / 3.11 / 3.12 and Poetry. You can then install from your local clone's root dir:
+To develop for Docling Core, you need Python 3.9 / 3.10 / 3.11 / 3.12 / 3.13 and Poetry. You can then install from your local clone's root dir:
 ```bash
 poetry install
 ```
@@ -81,14 +81,14 @@ poetry run pytest test
   Document.model_validate_json(data_str)
   ```
 
-- You can generate the JSON schema of a model with the script `ds_generate_jsonschema`.
+- You can generate the JSON schema of a model with the script `generate_jsonschema`.
 
   ```py
   # for the `Document` type
-  ds_generate_jsonschema Document
+  generate_jsonschema Document
 
   # for the use `Record` type
-  ds_generate_jsonschema Record
+  generate_jsonschema Record
   ```
 
 ## Documentation
@@ -97,12 +97,12 @@ Docling supports 3 main data types:
 
 - **Document** for publications like books, articles, reports, or patents. When Docling converts an unstructured PDF document, the generated JSON follows this schema.
   The Document type also models the metadata that may be attached to the converted document.
-  Check [Document](docs/Document.md) for the full JSON schema. 
+  Check [Document](docs/Document.json) for the full JSON schema. 
 - **Record** for structured database records, centered on an entity or _subject_ that is provided with a list of attributes.
   Related to records, the statements can represent annotations on text by Natural Language Processing (NLP) tools.
-  Check [Record](docs/Record.md) for the full JSON schema. 
+  Check [Record](docs/Record.json) for the full JSON schema. 
 - **Generic** for any data representation, ensuring minimal configuration and maximum flexibility.
-  Check [Generic](docs/Generic.md) for the full JSON schema. 
+  Check [Generic](docs/Generic.json) for the full JSON schema. 
 
 The data schemas are defined using [pydantic](https://pydantic-docs.helpmanual.io/) models, which provide built-in processes to support the creation of data that adhere to those models.
 

{docling_core-1.7.2.dist-info → docling_core-2.0.0.dist-info}/RECORD

@@ -14,31 +14,24 @@ docling_core/search/mapping.py,sha256=6rqG7LgYSeWmooKNEcRa5gFDLp1ZdzPqDGlwTA5gpO
 docling_core/search/meta.py,sha256=wSurrsqdP1N3gQKx027fVdzVmc33a7Y6rPl-FClQvtA,3318
 docling_core/search/package.py,sha256=Lz2ml2eDy5t0ZimnGTq-DXHAn-f18w0bn4H5xrhs75A,1841
 docling_core/transforms/__init__.py,sha256=P81y_oqkiTN4Ld5crh1gQ6BbHqqR6C6nBt9ACDd57ds,106
-docling_core/transforms/chunker/__init__.py,sha256=xZ5ELOB8tbCoJY1dKUvOrFqxYyoHmmCNUSHxrrRi8a4,317
-docling_core/transforms/chunker/base.py,sha256=5EW89CZf4SMB6Eh4yNjzYoNjn8S7oHH8NEpMck3Lcio,2078
-docling_core/transforms/chunker/hierarchical_chunker.py,sha256=DexBEPMR5rqnwrCMi-g98AtLDG7PKyZuf7u3NuXo-tA,12682
-docling_core/transforms/id_generator/__init__.py,sha256=7UoSyAcLsvw-RRrNjYXRVS4rIOUXjwqVpaQA-SSeINU,379
-docling_core/transforms/id_generator/base.py,sha256=SufPsaZUfMpuITq7pMv5YtlLmtGTDgA4LWmjmhQuSM0,704
-docling_core/transforms/id_generator/doc_hash_id_generator.py,sha256=SUw4FBhMZtbWCfc7oMucSwYvJTXqIPMn3yCXPRxtPCI,656
-docling_core/transforms/id_generator/uuid_generator.py,sha256=t8Bky_1JQB9myX-PJGWvW_c4-NvtHPHab6b1NdS-bpU,929
-docling_core/transforms/metadata_extractor/__init__.py,sha256=q_eAUcbaToEuYUPco4uiBO8vgTGSmZUC-r0mS7KbWh8,335
-docling_core/transforms/metadata_extractor/base.py,sha256=7h_S6-buCVtvAvKQKLISjDqFV8D3brewiQ-geqlUriI,1467
-docling_core/transforms/metadata_extractor/simple_metadata_extractor.py,sha256=ZRjDdXgFe8jPBNC_0ruJjQanabpkxceVsCJVVWVWlIg,1629
-docling_core/types/__init__.py,sha256=6mrAEKRW85uHJwNQBufwjPcMWCjm3oocA6MaO4_NLgg,805
+docling_core/transforms/chunker/__init__.py,sha256=cSY_2L6EpR0lkPSDgt_ikjVoQpgIAhofvBfvfR3w_1Y,270
+docling_core/transforms/chunker/base.py,sha256=uPNj6NHUl394Uh6wf01vmro4i3Ez4WUlV5ljfp85EM4,1565
+docling_core/transforms/chunker/hierarchical_chunker.py,sha256=tKJnaKhdAAOkwRu4WOoHISo7qgBx_4T3YNS4nPB_iqc,6390
+docling_core/types/__init__.py,sha256=x_IgCOzOr5TiOVrKNPjWIJjLkDqkyMwNSpDMkGPg2fg,872
 docling_core/types/base.py,sha256=PusJskRVL19y-hq0BgXr5e8--QEqSqLnFNJ8UbOqW88,8318
-docling_core/types/doc/__init__.py,sha256=Pzj_8rft6SJTVTCHgXRwHtuZjL6LK_6dcBWjikL9biY,125
-docling_core/types/doc/base.py,sha256=ujko-oQKoXw6wjBn0Il2Khu3PyljHqYnUNh3mPDVJF8,14676
-docling_core/types/doc/doc_ann.py,sha256=8pV2efUglw19jxl4_oqB__mSxjWvtGIcllyCdqA-b2s,1196
-docling_core/types/doc/doc_ocr.py,sha256=6PC0C-OczF-MyfgRxEI1xs3PWgNOzi7i2yEQbTqZz0I,1387
-docling_core/types/doc/doc_raw.py,sha256=Y69G6IiauNDaoT-5el4xo1ypWpnBJQ75akGGkCMTZSc,3888
-docling_core/types/doc/document.py,sha256=AKp1kOo0tncf9FX3q7qRWQ2Jz_hZE44smZpyrtsRzY4,24104
-docling_core/types/doc/tokens.py,sha256=uU_MYW_p7ypf7eYICFBvxdnVaPZ7CQnvZmbJ6oPrtEA,6134
-docling_core/types/experimental/__init__.py,sha256=mpqa2soTcHHEKqkcSeYBbAHepg0OgVZNReKvPmGz2r4,587
-docling_core/types/experimental/base.py,sha256=k04zvzNI7qo4HfKxLPCePKxCnerzXd582gvrVjF25SI,4225
-docling_core/types/experimental/document.py,sha256=X3z4sjRmWytRbEmSCnKat4D9sYxSV7Olm1YNmG3c5Kg,37874
-docling_core/types/experimental/labels.py,sha256=tpmvpmJuQyYMLhxAvJSVuFhDRh_zQNiP1WrQmNXKQzo,1224
+docling_core/types/doc/__init__.py,sha256=_6QvDWO_AV0iHx72PpDb6XLZTlA7KYQhfL80xGiCq70,625
+docling_core/types/doc/base.py,sha256=tNEXzxe2ihduCezYTUy_jNKMs0RJ6hBS79epYwyc2QY,4326
+docling_core/types/doc/document.py,sha256=hJxpmQOxfTQDL78lNp1LT-ap3uRcxFHKRXj7swscCGo,45403
+docling_core/types/doc/labels.py,sha256=mzmSd072A-qW3IThswHxwIHV8IoyTCbHHlNOrisinRA,1335
 docling_core/types/gen/__init__.py,sha256=C6TuCfvpSnSL5XDOFMcYHUY2-i08vvfOGRcdu6Af0pI,124
 docling_core/types/gen/generic.py,sha256=l4CZ4_Lb8ONG36WNJWbKX5hGKvTh_yU-hXp5hsm7uVU,844
+docling_core/types/legacy_doc/__init__.py,sha256=Pzj_8rft6SJTVTCHgXRwHtuZjL6LK_6dcBWjikL9biY,125
+docling_core/types/legacy_doc/base.py,sha256=l8NKCuORUQ1ebjdGWpj6b30oQEvtErLsIHKQHbbJiPg,14683
+docling_core/types/legacy_doc/doc_ann.py,sha256=CIQHW8yzu70bsMR9gtu7dqe4oz603Tq2eDDt9sh-tYo,1203
+docling_core/types/legacy_doc/doc_ocr.py,sha256=FfFqHAyMSbFt5cKeE7QLcxS0qUweBilBJoN9CH2TsQs,1394
+docling_core/types/legacy_doc/doc_raw.py,sha256=LrvQ9DhNjBRy98p_F9PUyHZeTGAxMKWqJzY4WJ7v-xs,3895
+docling_core/types/legacy_doc/document.py,sha256=A_cTYOjx6pNIICpOUm09YsfwPrIGDEZTKdetb2fx4PM,24273
+docling_core/types/legacy_doc/tokens.py,sha256=uU_MYW_p7ypf7eYICFBvxdnVaPZ7CQnvZmbJ6oPrtEA,6134
 docling_core/types/nlp/__init__.py,sha256=hGcztAeVK7xkRBqRRvc4zbY4PGeJ0r0QrEsetnSx9nI,119
 docling_core/types/nlp/qa.py,sha256=TyZjubqkEoREv0YzmuLKlq4WW_TnJNj7BoBY1_r2a1E,2731
 docling_core/types/nlp/qa_labels.py,sha256=YLW2SYM9M1riktCUYctsg83Msb988NV2I754w4ibWzA,5880
@@ -48,16 +41,16 @@ docling_core/types/rec/base.py,sha256=jhTfInNGyB9NUw7o33PElrFGL80TqhU8MLcLZNZYj3
 docling_core/types/rec/predicate.py,sha256=4iDwXl9c4jzHTDIlRNE88yvDzKA9_od0xjPUUUP5IjI,3959
 docling_core/types/rec/record.py,sha256=r1QgPepwH3YjmMHlwwmeK00ZHEJnAsvyOMeXFY_D9_Q,2750
 docling_core/types/rec/statement.py,sha256=YwcV4CbVaAbzNwh14yJ_6Py3Ww0XnUJrEEUiKRdCZ5o,1701
-docling_core/types/rec/subject.py,sha256=wX9qsihwDbR7ZNSzY3vQymxi0eN1nxxsonrhSZzsMhA,2565
+docling_core/types/rec/subject.py,sha256=PRCERGTMs4YhR3_Ne6jogkm41zYg8uUWb1yFpM7atm4,2572
 docling_core/utils/__init__.py,sha256=VauNNpWRHG0_ISKrsy5-gTxicrdQZSau6qMfuMl3iqk,120
 docling_core/utils/alias.py,sha256=B6Lqvss8CbaNARHLR4qSmNh9OkB6LvqTpxfsFmkLAFo,874
-docling_core/utils/ds_generate_docs.py,sha256=0xGBagdC_PGjyeHXYZo90VnVrSTMZgHb0SYhFa6X7bQ,4248
-docling_core/utils/ds_generate_jsonschema.py,sha256=EhNQutqWJFWuN-yl9UUPFZ7DJTvGqg54qBIvUMHTHdA,1647
 docling_core/utils/file.py,sha256=VQgzjyvmJnAIHB6ex7ikcmbDAR4GA1ALreuO7Ubrp50,1895
+docling_core/utils/generate_docs.py,sha256=td16gmPMAKk_Q-uAWIdYRGwWN9QRr_8_JQJM1gAF5Bw,2283
+docling_core/utils/generate_jsonschema.py,sha256=uNX1O5XnjyB5nA66XqZXTt3YbGuR2tyi_OhHepHYtZg,1654
 docling_core/utils/validate.py,sha256=3FmnxnKTDZC5J9OGxCL3U3DGRl0t0bBV1NcySXswdas,2031
-docling_core/utils/validators.py,sha256=fBdyWX4PvFh7o_d25ZTs4iwmeo75QTbrxsvXv2kXkTg,2777
-docling_core-1.7.2.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
-docling_core-1.7.2.dist-info/METADATA,sha256=OfpdHwn-55a4Z-61sx7SZ1yD7jMDaiZ_LEmEIoYKa9I,5383
-docling_core-1.7.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling_core-1.7.2.dist-info/entry_points.txt,sha256=XHhtJEkdUuLxXSNxLdFIzx_siQ3z2UFQEKp-P8VYAE4,189
-docling_core-1.7.2.dist-info/RECORD,,
+docling_core/utils/validators.py,sha256=azcrndLzhNkTWnbFSu9shJ5D3j_znnLrIFA5R8hzmGU,2798
+docling_core-2.0.0.dist-info/LICENSE,sha256=2M9-6EoQ1sxFztTOkXGAtwUDJvnWaAHdB9BYWVwGkIw,1087
+docling_core-2.0.0.dist-info/METADATA,sha256=LUzVQtKKrKcVKRlKtI_oWGaKMYt08h0poFGlnxF2zg0,5389
+docling_core-2.0.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling_core-2.0.0.dist-info/entry_points.txt,sha256=jIxlWv3tnO04irlZc0zfhqJIgz1bg9Hha4AkaLWSdUA,177
+docling_core-2.0.0.dist-info/RECORD,,

docling_core-2.0.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+generate_docs=docling_core.utils.generate_docs:main
+generate_jsonschema=docling_core.utils.generate_jsonschema:main
+validate=docling_core.utils.validate:main
+

docling_core/transforms/id_generator/__init__.py

@@ -1,12 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Define the ID generator types."""
-
-from docling_core.transforms.id_generator.base import BaseIDGenerator  # noqa
-from docling_core.transforms.id_generator.doc_hash_id_generator import (  # noqa
-    DocHashIDGenerator,
-)
-from docling_core.transforms.id_generator.uuid_generator import UUIDGenerator  # noqa

docling_core/transforms/id_generator/base.py

@@ -1,30 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Base document ID generator module."""
-
-from abc import ABC, abstractmethod
-from typing import Any
-
-from docling_core.types import Document as DLDocument
-
-
-class BaseIDGenerator(ABC):
-    """Document ID generator base class."""
-
-    @abstractmethod
-    def generate_id(self, doc: DLDocument, *args: Any, **kwargs: Any) -> str:
-        """Generate an ID for the given document.
-
-        Args:
-            doc (DLDocument): document to generate ID for
-
-        Raises:
-            NotImplementedError: in this abstract implementation
-
-        Returns:
-            str: the generated ID
-        """
-        raise NotImplementedError()

docling_core/transforms/id_generator/doc_hash_id_generator.py

@@ -1,27 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Doc-hash-based ID generator module."""
-
-
-from typing import Any
-
-from docling_core.transforms.id_generator import BaseIDGenerator
-from docling_core.types import Document as DLDocument
-
-
-class DocHashIDGenerator(BaseIDGenerator):
-    """Doc-hash-based ID generator class."""
-
-    def generate_id(self, doc: DLDocument, *args: Any, **kwargs: Any) -> str:
-        """Generate an ID for the given document.
-
-        Args:
-            doc (DLDocument): document to generate ID for
-
-        Returns:
-            str: the generated ID
-        """
-        return doc.file_info.document_hash

docling_core/transforms/id_generator/uuid_generator.py

@@ -1,34 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""UUID-based ID generator module."""
-
-from random import Random
-from typing import Annotated, Any, Optional
-from uuid import UUID
-
-from pydantic import BaseModel, Field
-
-from docling_core.transforms.id_generator import BaseIDGenerator
-from docling_core.types import Document as DLDocument
-
-
-class UUIDGenerator(BaseModel, BaseIDGenerator):
-    """UUID-based ID generator class."""
-
-    seed: Optional[int] = None
-    uuid_version: Annotated[int, Field(strict=True, ge=1, le=5)] = 4
-
-    def generate_id(self, doc: DLDocument, *args: Any, **kwargs: Any) -> str:
-        """Generate an ID for the given document.
-
-        Args:
-            doc (DLDocument): document to generate ID for
-
-        Returns:
-            str: the generated ID
-        """
-        rd = Random(x=self.seed)
-        return str(UUID(int=rd.getrandbits(128), version=self.uuid_version))

docling_core/transforms/metadata_extractor/__init__.py

@@ -1,13 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Define the metadata extractor types."""
-
-from docling_core.transforms.metadata_extractor.base import (  # noqa
-    BaseMetadataExtractor,
-)
-from docling_core.transforms.metadata_extractor.simple_metadata_extractor import (  # noqa
-    SimpleMetadataExtractor,
-)

docling_core/transforms/metadata_extractor/base.py

@@ -1,59 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Base metadata extractor module."""
-
-
-from abc import ABC, abstractmethod
-from typing import Any
-
-from pydantic import BaseModel
-
-from docling_core.types import Document as DLDocument
-
-
-class BaseMetadataExtractor(BaseModel, ABC):
-    """Metadata extractor base class."""
-
-    @abstractmethod
-    def get_metadata(
-        self, doc: DLDocument, *args: Any, **kwargs: Any
-    ) -> dict[str, Any]:
-        """Extract metadata for the given document.
-
-        Args:
-            doc (DLDocument): document to extract metadata for
-
-        Raises:
-            NotImplementedError: in this abstract implementation
-
-        Returns:
-            dict[str, Any]: the extracted metadata
-        """
-        raise NotImplementedError()
-
-    @abstractmethod
-    def get_excluded_embed_metadata_keys(self) -> list[str]:
-        """Get metadata keys to exclude from embedding.
-
-        Raises:
-            NotImplementedError: in this abstract implementation
-
-        Returns:
-            list[str]: the metadata to exclude
-        """
-        raise NotImplementedError()
-
-    @abstractmethod
-    def get_excluded_llm_metadata_keys(self) -> list[str]:
-        """Get metadata keys to exclude from LLM generation.
-
-        Raises:
-            NotImplementedError: in this abstract implementation
-
-        Returns:
-            list[str]: the metadata to exclude
-        """
-        raise NotImplementedError()

docling_core/transforms/metadata_extractor/simple_metadata_extractor.py

@@ -1,59 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Simple metadata extractor module."""
-
-
-from typing import Any, Final
-
-from docling_core.transforms.metadata_extractor import BaseMetadataExtractor
-from docling_core.types import Document as DLDocument
-
-_DL_DOC_HASH: Final[str] = "dl_doc_hash"
-_ORIGIN: Final[str] = "origin"
-
-
-class SimpleMetadataExtractor(BaseMetadataExtractor):
-    """Simple metadata extractor class."""
-
-    include_origin: bool = False
-
-    def get_metadata(
-        self, doc: DLDocument, origin: str, *args: Any, **kwargs: Any
-    ) -> dict[str, Any]:
-        """Extract metadata for the given document.
-
-        Args:
-            doc (DLDocument): document to extract metadata for
-            origin (str): the document origin
-
-        Returns:
-            dict[str, Any]: the extracted metadata
-        """
-        meta: dict[str, Any] = {
-            _DL_DOC_HASH: doc.file_info.document_hash,
-        }
-        if self.include_origin:
-            meta[_ORIGIN] = origin
-        return meta
-
-    def get_excluded_embed_metadata_keys(self) -> list[str]:
-        """Get metadata keys to exclude from embedding.
-
-        Returns:
-            list[str]: the metadata to exclude
-        """
-        excl_keys: list[str] = [_DL_DOC_HASH]
-        if self.include_origin:
-            excl_keys.append(_ORIGIN)
-        return excl_keys
-
-    def get_excluded_llm_metadata_keys(self) -> list[str]:
-        """Get metadata keys to exclude from LLM generation.
-
-        Returns:
-            list[str]: the metadata to exclude
-        """
-        return self.get_excluded_embed_metadata_keys()

docling_core/types/experimental/__init__.py

@@ -1,30 +0,0 @@
-#
-# Copyright IBM Corp. 2024 - 2024
-# SPDX-License-Identifier: MIT
-#
-
-"""Package for models defined by the Document type."""
-
-from .base import BoundingBox, CoordOrigin, Size
-from .document import (
-    BasePictureData,
-    BaseTableData,
-    DescriptionItem,
-    DocItem,
-    DoclingDocument,
-    DocumentOrigin,
-    FloatingItem,
-    GroupItem,
-    ImageRef,
-    KeyValueItem,
-    NodeItem,
-    PageItem,
-    PictureItem,
-    ProvenanceItem,
-    RefItem,
-    SectionHeaderItem,
-    TableCell,
-    TableItem,
-    TextItem,
-)
-from .labels import DocItemLabel, GroupLabel, TableCellLabel