document2graph 0.1.0__tar.gz

document2graph-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Marina Walther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

document2graph-0.1.0/PKG-INFO

@@ -0,0 +1,243 @@
+Metadata-Version: 2.4
+Name: document2graph
+Version: 0.1.0
+Summary: Extract hierarchical document graphs and baseline chunks from PDF
+Author-email: Marina Walther <marina.walther@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/mwalther10/document2graph
+Project-URL: Issues, https://github.com/mwalther10/document2graph/issues
+Keywords: pdf,document2graph,docling,networkx,chunking,retrieval
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing :: Indexing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: docling
+Requires-Dist: docling-core
+Requires-Dist: docling-parse
+Requires-Dist: numpy
+Requires-Dist: pydantic>=2
+Requires-Dist: regex
+Requires-Dist: networkx
+Requires-Dist: coloredlogs
+Requires-Dist: tqdm
+Requires-Dist: transformers
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: httpx; extra == "dev"
+Dynamic: license-file
+
+# document2graph
+
+Extract hierarchical document graphs and baseline chunks from PDF files. The package parses PDFs using [Docling](https://github.com/DS4SD/docling) and produces two complementary representations:
+
+- **Document graph** — text, image, and table nodes connected by weighted structural edges, capturing the layout hierarchy of the document. Every graph is guaranteed to be connected: nodes without a resolvable parent are attached to a synthetic document root node.
+- **Baseline chunks** — flat, semantically merged chunks suitable for retrieval pipelines, with optional context enrichment.
+
+## Installation
+
+```bash
+pip install document2graph
+```
+
+Or from source:
+
+```bash
+pip install -e .
+```
+
+For development dependencies (pytest):
+
+```bash
+pip install -e ".[dev]"
+```
+
+Requires Python 3.11+.
+
+## Quick start
+
+### Configuration
+
+Both extractors share the same `ExtractorConfig`:
+
+```python
+from document2graph.models import ExtractorConfig
+from docling.datamodel.pipeline_options import PdfPipelineOptions
+
+config = ExtractorConfig(
+    pdf_path="data/pdfs/",        # directory containing input PDFs
+    data_path="data/output/",     # root directory for all output files
+    save_json=True,               # write results to disk as JSON (default: True)
+    document_type="report",       # arbitrary label stored in document metadata
+    pdfPipelineOptions=PdfPipelineOptions(),  # optional Docling pipeline config
+)
+```
+
+#### Metadata extraction
+
+The `metadata_config` field controls how document metadata (title, authors, version, etc.) is
+located inside each PDF. Each field is described by a search string and an inclusive page range
+to search on. Fields set to `None` are skipped.
+
+```python
+from document2graph.models import ExtractorConfig, MetadataExtractionConfig, MetadataFieldConfig
+
+config = ExtractorConfig(
+    pdf_path="data/pdfs/",
+    data_path="data/output/",
+    metadata_config=MetadataExtractionConfig(
+        title_page=1,
+        version=MetadataFieldConfig(label="edition", pages=(1, 1)),
+        authors=MetadataFieldConfig(label="authors", pages=(1, 2)),
+        institutions=MetadataFieldConfig(label="affiliations", pages=(1, 2)),
+        bibliography=None,        # not present in this document type
+        correspondence=MetadataFieldConfig(label="contact", pages=(2, 3)),
+    ),
+)
+```
+
+The default `MetadataExtractionConfig()` targets German-language Praxisempfehlungen PDFs
+(version on page 1, authors/institutions/bibliography/correspondence on page 2).
+
+#### Edge weights
+
+Every edge in the document graph carries a `weight` attribute reflecting the strength of the
+structural connection. The defaults can be overridden per category via `edge_weights`:
+
+```python
+from document2graph.models import ExtractorConfig, EdgeWeightConfig
+
+config = ExtractorConfig(
+    pdf_path="data/pdfs/",
+    data_path="data/output/",
+    edge_weights=EdgeWeightConfig(
+        section=1.0,             # heading -> subheading
+        text=0.8,                # heading/body -> body text, subtext, footnotes
+        list_item=0.6,           # anchor text -> grouped list item (bullets)
+        media=0.9,               # referencing text -> image/table (matched via caption)
+        unreferenced_media=0.3,  # fallback heading -> image/table without a caption match
+        root=0.1,                # document root -> otherwise disconnected node
+    ),
+)
+```
+
+#### Connectivity guarantee
+
+Each graph contains a synthetic root node (`#/document-root`, labelled with the document
+title). Any node whose parent cannot be resolved — as well as any component that would
+otherwise be disconnected — is attached to this root with the `root` edge weight, so every
+document graph is a single (weakly) connected component.
+
+Extracted metadata is available on the `Document` object under the `metadata` sub-object:
+
+```python
+result = extractor.run("my_document.pdf")
+doc = result["document_metadata"]
+
+print(doc.title)
+print(doc.metadata.authors)
+print(doc.metadata.version)
+```
+
+### Document graph extraction
+
+Processes every PDF in `pdf_path` and builds a NetworkX graph per document.
+
+```python
+from document2graph.document2graph_extractor import DocumentGraphExtractor
+
+extractor = DocumentGraphExtractor(config)
+
+# Process all PDFs and save graphs + snippets to disk
+extractor.generate_snippets()
+
+# Or collect Snippet objects in memory as well
+snippets = extractor.generate_snippets(return_snippets=True)
+```
+
+To process a single file and get the raw graph components:
+
+```python
+result = extractor.run("my_document.pdf")
+
+text_nodes   = result["text_nodes"]
+image_nodes  = result["image_nodes"]
+table_nodes  = result["table_nodes"]
+edges        = result["edges"]        # list of (parent_id, child_id, weight) tuples
+metadata     = result["document_metadata"]
+```
+
+Graphs are saved as `.gexf` files under `<data_path>/nx_graphs/` and can be loaded with NetworkX:
+
+```python
+import networkx as nx
+G = nx.read_gexf("data/output/nx_graphs/my_document.gexf")
+```
+
+### Baseline chunk extraction
+
+Produces flat, retrieval-ready chunks using Docling's hybrid chunker.
+
+```python
+from document2graph.baseline_extractor import BaselineExtractor
+
+extractor = BaselineExtractor(config)
+
+# Process all PDFs; saves JSON to disk
+extractor.generate_baseline_chunks()
+
+# Or return chunks in memory
+all_chunks = extractor.generate_baseline_chunks(return_chunks=True)
+# all_chunks["my_document.pdf"]["baseline"]  -> List[Chunk]
+# all_chunks["my_document.pdf"]["enriched"]  -> List[Chunk] (context-enriched)
+```
+
+To process a single file:
+
+```python
+chunks = extractor.extract_baseline_chunks(
+    filename="my_document.pdf",
+    baseline_description="Q1 report chunks",
+    enrich=True,   # also produce context-enriched variants
+)
+```
+
+## Output structure
+
+```
+<data_path>/
+├── raw_texts/          # intermediate Docling extraction output
+├── nx_graphs/          # .gexf graph files (one per document)
+├── snippets/           # <filename>_snippets.json (document graph output)
+└── baseline_chunks/    # <filename>_baseline_chunks.json
+```
+
+## Data models
+
+| Model | Key fields |
+|---|---|
+| `Snippet` | `snippet_id`, `type` (text/image/table), `document_id`, `sequence_no`, `label`, `level`, `page_no`, `bbox`, `text` |
+| `Chunk` | `chunk_id`, `document_id`, `text`, `meta`, `baseline_description`, `embedding` |
+| `Document` | `document_id`, `document_type`, `filename`, `title`, `metadata: DocumentMetadata` |
+| `DocumentMetadata` | `version`, `authors`, `institutions`, `bibliography`, `correspondence` |
+| `MetadataExtractionConfig` | `title_page`, `version`, `authors`, `institutions`, `bibliography`, `correspondence` (each a `MetadataFieldConfig`) |
+| `MetadataFieldConfig` | `label` (search string), `pages` (inclusive 1-based page range, e.g. `(1, 3)`) |
+| `EdgeWeightConfig` | `section`, `text`, `list_item`, `media`, `unreferenced_media`, `root` (edge weights by category) |
+
+## Running tests
+
+```bash
+pytest
+```
+
+Skip slow tests that download models:
+
+```bash
+pytest -m "not slow"
+```

document2graph-0.1.0/README.md

@@ -0,0 +1,208 @@
+# document2graph
+
+Extract hierarchical document graphs and baseline chunks from PDF files. The package parses PDFs using [Docling](https://github.com/DS4SD/docling) and produces two complementary representations:
+
+- **Document graph** — text, image, and table nodes connected by weighted structural edges, capturing the layout hierarchy of the document. Every graph is guaranteed to be connected: nodes without a resolvable parent are attached to a synthetic document root node.
+- **Baseline chunks** — flat, semantically merged chunks suitable for retrieval pipelines, with optional context enrichment.
+
+## Installation
+
+```bash
+pip install document2graph
+```
+
+Or from source:
+
+```bash
+pip install -e .
+```
+
+For development dependencies (pytest):
+
+```bash
+pip install -e ".[dev]"
+```
+
+Requires Python 3.11+.
+
+## Quick start
+
+### Configuration
+
+Both extractors share the same `ExtractorConfig`:
+
+```python
+from document2graph.models import ExtractorConfig
+from docling.datamodel.pipeline_options import PdfPipelineOptions
+
+config = ExtractorConfig(
+    pdf_path="data/pdfs/",        # directory containing input PDFs
+    data_path="data/output/",     # root directory for all output files
+    save_json=True,               # write results to disk as JSON (default: True)
+    document_type="report",       # arbitrary label stored in document metadata
+    pdfPipelineOptions=PdfPipelineOptions(),  # optional Docling pipeline config
+)
+```
+
+#### Metadata extraction
+
+The `metadata_config` field controls how document metadata (title, authors, version, etc.) is
+located inside each PDF. Each field is described by a search string and an inclusive page range
+to search on. Fields set to `None` are skipped.
+
+```python
+from document2graph.models import ExtractorConfig, MetadataExtractionConfig, MetadataFieldConfig
+
+config = ExtractorConfig(
+    pdf_path="data/pdfs/",
+    data_path="data/output/",
+    metadata_config=MetadataExtractionConfig(
+        title_page=1,
+        version=MetadataFieldConfig(label="edition", pages=(1, 1)),
+        authors=MetadataFieldConfig(label="authors", pages=(1, 2)),
+        institutions=MetadataFieldConfig(label="affiliations", pages=(1, 2)),
+        bibliography=None,        # not present in this document type
+        correspondence=MetadataFieldConfig(label="contact", pages=(2, 3)),
+    ),
+)
+```
+
+The default `MetadataExtractionConfig()` targets German-language Praxisempfehlungen PDFs
+(version on page 1, authors/institutions/bibliography/correspondence on page 2).
+
+#### Edge weights
+
+Every edge in the document graph carries a `weight` attribute reflecting the strength of the
+structural connection. The defaults can be overridden per category via `edge_weights`:
+
+```python
+from document2graph.models import ExtractorConfig, EdgeWeightConfig
+
+config = ExtractorConfig(
+    pdf_path="data/pdfs/",
+    data_path="data/output/",
+    edge_weights=EdgeWeightConfig(
+        section=1.0,             # heading -> subheading
+        text=0.8,                # heading/body -> body text, subtext, footnotes
+        list_item=0.6,           # anchor text -> grouped list item (bullets)
+        media=0.9,               # referencing text -> image/table (matched via caption)
+        unreferenced_media=0.3,  # fallback heading -> image/table without a caption match
+        root=0.1,                # document root -> otherwise disconnected node
+    ),
+)
+```
+
+#### Connectivity guarantee
+
+Each graph contains a synthetic root node (`#/document-root`, labelled with the document
+title). Any node whose parent cannot be resolved — as well as any component that would
+otherwise be disconnected — is attached to this root with the `root` edge weight, so every
+document graph is a single (weakly) connected component.
+
+Extracted metadata is available on the `Document` object under the `metadata` sub-object:
+
+```python
+result = extractor.run("my_document.pdf")
+doc = result["document_metadata"]
+
+print(doc.title)
+print(doc.metadata.authors)
+print(doc.metadata.version)
+```
+
+### Document graph extraction
+
+Processes every PDF in `pdf_path` and builds a NetworkX graph per document.
+
+```python
+from document2graph.document2graph_extractor import DocumentGraphExtractor
+
+extractor = DocumentGraphExtractor(config)
+
+# Process all PDFs and save graphs + snippets to disk
+extractor.generate_snippets()
+
+# Or collect Snippet objects in memory as well
+snippets = extractor.generate_snippets(return_snippets=True)
+```
+
+To process a single file and get the raw graph components:
+
+```python
+result = extractor.run("my_document.pdf")
+
+text_nodes   = result["text_nodes"]
+image_nodes  = result["image_nodes"]
+table_nodes  = result["table_nodes"]
+edges        = result["edges"]        # list of (parent_id, child_id, weight) tuples
+metadata     = result["document_metadata"]
+```
+
+Graphs are saved as `.gexf` files under `<data_path>/nx_graphs/` and can be loaded with NetworkX:
+
+```python
+import networkx as nx
+G = nx.read_gexf("data/output/nx_graphs/my_document.gexf")
+```
+
+### Baseline chunk extraction
+
+Produces flat, retrieval-ready chunks using Docling's hybrid chunker.
+
+```python
+from document2graph.baseline_extractor import BaselineExtractor
+
+extractor = BaselineExtractor(config)
+
+# Process all PDFs; saves JSON to disk
+extractor.generate_baseline_chunks()
+
+# Or return chunks in memory
+all_chunks = extractor.generate_baseline_chunks(return_chunks=True)
+# all_chunks["my_document.pdf"]["baseline"]  -> List[Chunk]
+# all_chunks["my_document.pdf"]["enriched"]  -> List[Chunk] (context-enriched)
+```
+
+To process a single file:
+
+```python
+chunks = extractor.extract_baseline_chunks(
+    filename="my_document.pdf",
+    baseline_description="Q1 report chunks",
+    enrich=True,   # also produce context-enriched variants
+)
+```
+
+## Output structure
+
+```
+<data_path>/
+├── raw_texts/          # intermediate Docling extraction output
+├── nx_graphs/          # .gexf graph files (one per document)
+├── snippets/           # <filename>_snippets.json (document graph output)
+└── baseline_chunks/    # <filename>_baseline_chunks.json
+```
+
+## Data models
+
+| Model | Key fields |
+|---|---|
+| `Snippet` | `snippet_id`, `type` (text/image/table), `document_id`, `sequence_no`, `label`, `level`, `page_no`, `bbox`, `text` |
+| `Chunk` | `chunk_id`, `document_id`, `text`, `meta`, `baseline_description`, `embedding` |
+| `Document` | `document_id`, `document_type`, `filename`, `title`, `metadata: DocumentMetadata` |
+| `DocumentMetadata` | `version`, `authors`, `institutions`, `bibliography`, `correspondence` |
+| `MetadataExtractionConfig` | `title_page`, `version`, `authors`, `institutions`, `bibliography`, `correspondence` (each a `MetadataFieldConfig`) |
+| `MetadataFieldConfig` | `label` (search string), `pages` (inclusive 1-based page range, e.g. `(1, 3)`) |
+| `EdgeWeightConfig` | `section`, `text`, `list_item`, `media`, `unreferenced_media`, `root` (edge weights by category) |
+
+## Running tests
+
+```bash
+pytest
+```
+
+Skip slow tests that download models:
+
+```bash
+pytest -m "not slow"
+```

document2graph-0.1.0/document2graph/baseline_extractor/__init__.py

@@ -0,0 +1,3 @@
+from .extractor import BaselineExtractor
+
+__all__ = ["BaselineExtractor"]

document2graph-0.1.0/document2graph/baseline_extractor/extractor.py

@@ -0,0 +1,82 @@
+import os
+import uuid
+
+from ..utils.hybrid_chunker_extractor import HybridChunkerExtractor
+from tqdm import tqdm
+import json
+from ..models.Chunk import Chunk
+from ..models.ExtractorConfig import ExtractorConfig
+from typing import List
+
+from ..utils.log import Log
+
+_log = Log("BaselineExtractor")
+
+class BaselineExtractor:
+    def __init__(self, config: ExtractorConfig):
+        self.logger = _log.logger
+        self.pdf_path = config.pdf_path
+        self.data_path = config.data_path
+        self.raw_text_save_dir = os.path.join(self.data_path, "raw_texts/")
+        self.pdfPipelineOptions = config.pdfPipelineOptions
+        self.save_json = config.save_json
+        self.document_type = config.document_type
+    
+    def _get_clean_filename(self, filename: str) -> str:
+        return os.path.splitext(os.path.basename(filename))[0]
+
+    def extract_baseline_chunks(self, filename: str, baseline_description: str, enrich: bool = False) -> dict[str, List[Chunk]]:
+        """Extract baseline chunks from the raw text files for a given document."""
+        sample = os.path.join(self.pdf_path, filename)
+        clean_filename = self._get_clean_filename(sample)
+        baseline_extractor = HybridChunkerExtractor(
+            source=sample,
+            merge_peers=True,
+            pipeline_options=self.pdfPipelineOptions
+        )
+        chunks = []
+        enriched_chunks = []
+        for chunk in baseline_extractor.extract_and_chunk(save_dir=self.raw_text_save_dir, filename=clean_filename):
+            meta = chunk.meta.export_json_dict()
+
+            chunks.append(Chunk(
+                chunk_id=str(uuid.uuid4()),
+                document_id=filename,
+                text=chunk.text,
+                meta=meta,
+                baseline_description=baseline_description
+            ))
+            if enrich:
+                enriched_text = baseline_extractor.chunker.contextualize(chunk)
+                enriched_chunks.append(Chunk(
+                    chunk_id=str(uuid.uuid4()),
+                    document_id=filename,
+                    text=enriched_text,
+                    meta=meta,
+                    baseline_description=f"Enriched; {baseline_description}"
+                ))
+        return {"baseline": chunks, "enriched": enriched_chunks}
+    
+    def generate_baseline_chunks(self, return_chunks: bool = False) -> dict[str, List[Chunk]] | None:
+        """Generate baseline chunks for all documents in the pdf_path."""
+        all_chunks = {}
+        for filename in tqdm(os.listdir(self.pdf_path)):
+            clean_filename = self._get_clean_filename(filename)
+            if filename.endswith(".pdf"):
+                self.logger.info(f"Extracting baseline chunks for {filename}...")
+                baseline_description = f"Baseline chunks for {filename}"
+                chunks = self.extract_baseline_chunks(filename, baseline_description, enrich=True)
+                all_chunks[filename] = chunks
+
+                # by default, the chunks are saved as json
+                if self.save_json:
+                    baseline_json = os.path.join(self.data_path, "baseline_chunks/")
+                    os.makedirs(baseline_json, exist_ok=True)
+                    with open(f"{self.raw_text_save_dir}/{clean_filename}_baseline_chunks.json", "w") as f:
+                        json.dump([chunk.model_dump() for chunk in chunks["baseline"]], f, indent=4)
+                        if len(chunks["enriched"]) > 0:
+                            with open(f"{self.raw_text_save_dir}/{clean_filename}_enriched_chunks.json", "w") as f:
+                                json.dump([chunk.model_dump() for chunk in chunks["enriched"]], f, indent=4)
+            
+
+        return all_chunks if return_chunks else None

document2graph-0.1.0/document2graph/document2graph_extractor/__init__.py

@@ -0,0 +1,3 @@
+from .extractor import DocumentGraphExtractor
+
+__all__ = ["DocumentGraphExtractor"]