docling-pipelines 1.0.0.dev1__tar.gz → 1.0.0.dev2__tar.gz

docling_pipelines-1.0.0.dev2/PKG-INFO

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: docling-pipelines
+Version: 1.0.0.dev2
+Summary: This repository contains the docpipe operators.
+Maintainer-email: Jojo Joseph <jjojo@in.ibm.com>, Aditya Rohan Singh <adityars@ibm.com>, Anudeep Dunaboyina <anudeep.dunaboyina@ibm.com>, Ashok Kumar Angadi <asangadi@in.ibm.com>, Harshad Nehate <Harshad.Nehate@ibm.com>, Kurian Kevin <Kurian.Kevin@ibm.com>, Nipun Bayas <nipun.bayas@ibm.com>, Paul Baby <Paul.Baby1@ibm.com>, Prasad Madugundu <smadugun@in.ibm.com>, Remya P <remya.p@ibm.com>, Sahil Berwal <Sahil.Berwal@ibm.com>, Sanchit Rote <Sanchit.Rote@ibm.com>, Yash Bopardikar <ybopard@us.ibm.com>, Benson Fernandes <benferna@in.ibm.com>, Manish Soni <Manish.Soni3@ibm.com>, Docling Pipeline Team <docling_pipeline_team-dg@ibm.com>
+License-File: LICENSE
+Requires-Python: <3.13,>=3.12
+Requires-Dist: accelerate==1.13.0
+Requires-Dist: aiohttp==3.14.1
+Requires-Dist: alembic==1.18.4
+Requires-Dist: azure-storage-blob==12.28.0
+Requires-Dist: babel==2.18.0
+Requires-Dist: boto3==1.42.55
+Requires-Dist: botocore==1.42.55
+Requires-Dist: box-sdk-gen==1.17.0
+Requires-Dist: data-prep-toolkit-transforms==1.1.8
+Requires-Dist: datatrove==0.3.0
+Requires-Dist: datefinder==1.0.0
+Requires-Dist: docling[rapidocr]==2.105.0
+Requires-Dist: duckdb==1.3.2
+Requires-Dist: fastapi==0.136.3
+Requires-Dist: fasttext-wheel==0.9.2
+Requires-Dist: filelock==3.24.2
+Requires-Dist: ftfy==6.3.1
+Requires-Dist: google-api-python-client==2.190.0
+Requires-Dist: google-auth-httplib2==0.3.0
+Requires-Dist: google-auth-oauthlib==1.2.4
+Requires-Dist: google-cloud-storage==3.9.0
+Requires-Dist: google-re2==1.1.20251105
+Requires-Dist: griffe==1.15.0
+Requires-Dist: httpx==0.28.1
+Requires-Dist: ibm-cos-sdk==2.14.3
+Requires-Dist: ibm-watsonx-ai==1.5.10
+Requires-Dist: idna==3.15
+Requires-Dist: langchain-classic==1.0.7
+Requires-Dist: langchain-community==0.4.1
+Requires-Dist: langchain-core==1.4.6
+Requires-Dist: langchain-experimental==0.4.1
+Requires-Dist: langchain-text-splitters==1.1.2
+Requires-Dist: langchain==1.3.9
+Requires-Dist: langdetect==1.0.9
+Requires-Dist: langsmith==0.8.4
+Requires-Dist: litellm==1.84.0
+Requires-Dist: lxml==6.1.0
+Requires-Dist: mako==1.3.12
+Requires-Dist: mlx==0.30.6; sys_platform == 'darwin'
+Requires-Dist: msal==1.37.0
+Requires-Dist: nltk==3.10.0
+Requires-Dist: o365==2.1.9
+Requires-Dist: ollama==0.6.1
+Requires-Dist: openai==2.33.0
+Requires-Dist: opensearch-py==3.1.0
+Requires-Dist: orjson==3.11.7
+Requires-Dist: pandas==2.3.3
+Requires-Dist: passlib[bcrypt]==1.7.4
+Requires-Dist: pillow==12.2.0
+Requires-Dist: polars==1.32.3
+Requires-Dist: prefect==3.7.1
+Requires-Dist: psycopg2-binary==2.9.12
+Requires-Dist: pyarrow==24.0.0
+Requires-Dist: pyasn1==0.6.3
+Requires-Dist: pydantic-settings==2.14.0
+Requires-Dist: pydantic==2.12.5
+Requires-Dist: pygments==2.20.0
+Requires-Dist: pyiceberg-core==0.7.0
+Requires-Dist: pyiceberg[glue]==0.9.1
+Requires-Dist: pyjwt==2.13.0
+Requires-Dist: pymilvus==2.6.13
+Requires-Dist: pyopenssl==26.3.0
+Requires-Dist: pypdf==6.13.3
+Requires-Dist: pyphen==0.17.2
+Requires-Dist: python-dotenv==1.2.1
+Requires-Dist: python-jose[cryptography]==3.5.0
+Requires-Dist: python-ldap==3.4.5
+Requires-Dist: pyyaml==6.0.3
+Requires-Dist: requests==2.32.5
+Requires-Dist: sentence-transformers==5.4.1
+Requires-Dist: setuptools==81.0.0
+Requires-Dist: sqlalchemy==2.0.49
+Requires-Dist: sqlglot==27.13.2
+Requires-Dist: sqlmodel==0.0.38
+Requires-Dist: starlette==1.3.1
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: tenacity==8.3.0
+Requires-Dist: toml==0.10.2
+Requires-Dist: torch==2.12.1
+Requires-Dist: transformers==5.8.1
+Requires-Dist: unicategories==0.1.2
+Requires-Dist: unicodedataplus==16.0.0.post1
+Requires-Dist: urllib3==2.7.0
+Requires-Dist: uvicorn[standard]==0.40.0
+Provides-Extra: dev
+Requires-Dist: black==26.1.0; extra == 'dev'
+Requires-Dist: detect-secrets==1.5.0; extra == 'dev'
+Requires-Dist: faker==40.8.0; extra == 'dev'
+Requires-Dist: flake8==7.3.0; extra == 'dev'
+Requires-Dist: httpx==0.28.1; extra == 'dev'
+Requires-Dist: mypy==1.19.1; extra == 'dev'
+Requires-Dist: pre-commit==4.5.1; extra == 'dev'
+Requires-Dist: pytest-asyncio==1.4.0; extra == 'dev'
+Requires-Dist: pytest-cov==7.0.0; extra == 'dev'
+Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
+Requires-Dist: pytest==9.0.2; extra == 'dev'
+Requires-Dist: ruff==0.15.5; extra == 'dev'
+Provides-Extra: notebooks
+Requires-Dist: ipykernel==7.3.0; extra == 'notebooks'
+Requires-Dist: jupyter==1.1.1; extra == 'notebooks'
+Requires-Dist: matplotlib==3.11.0; extra == 'notebooks'
+Requires-Dist: nbconvert==7.17.1; extra == 'notebooks'
+Requires-Dist: nbformat==5.10.4; extra == 'notebooks'
+Requires-Dist: seaborn==0.13.2; extra == 'notebooks'
+Provides-Extra: vlm-asr
+Requires-Dist: docling[asr,vlm]; extra == 'vlm-asr'
+Description-Content-Type: text/markdown
+
+# Docling pipelines
+
+[![PyPI version](https://img.shields.io/pypi/v/docling-pipelines)](https://pypi.org/project/docling-pipelines/)
+[![Python 3.12](https://img.shields.io/badge/python-3.12-blue.svg)](https://www.python.org/downloads/release/python-3120/)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![License MIT](https://img.shields.io/github/license/IBM/docling-pipelines)](https://opensource.org/licenses/MIT)
+
+## What is Docling pipelines?
+
+Docling pipelines is an enterprise-grade document curation pipeline for Retrieval Augmented Generation (RAG) applications. It ingests data from unstructured sources, curates documents, and writes entities and vector embeddings to targets — enabling AI-ready pipelines at scale.
+
+It connects to cloud document sources (S3, OneDrive, SharePoint, Google Drive, Box, and more) and extracts content and entities from PDF, DOCX, HTML, images, and other formats using [Docling](https://github.com/docling-project/docling). Extracted content is curated for LLMs, converted into chunks and embeddings, and stored in a vector database such as Milvus or OpenSearch.
+
+## Features
+
+- 📥 **Multi-source ingestion** — local filesystem, Amazon S3, IBM COS, SharePoint, OneDrive, Google Drive, Box, CSV, and web pages
+- 📄 **Document extraction** — PDF, DOCX, HTML, images, and more via Docling, with optional VLM and ASR pipelines
+- 🧠 **Entity extraction** — LLM-based extraction via LiteLLM (100+ providers), IBM watsonx.ai, or Docling templates
+- ✂️ **Chunking** — Docling-native and semantic chunking strategies
+- 🔢 **Embeddings** — vector embedding generation for any downstream vector store
+- 🔍 **Quality operators** — language detection, readability scoring, PII/HAP detection, deduplication, redaction, SQL filtering, document classification, and ML enrichment
+- 🗄️ **Vector storage** — write to OpenSearch or Milvus
+- 🔀 **DAG-based flows** — define pipelines as JSON with automatic dependency resolution and parallel execution
+- 🔌 **Extensible** — load custom operators from Python packages, local paths, or S3 without modifying core code
+- 🖥️ **Multiple interfaces** — CLI, Python API (`DocpipeFlowManager`), and REST API (FastAPI)
+
+## Quickstart
+
+### 1. Install
+
+```bash
+pip install docling-pipelines
+```
+
+Requires Python 3.12. Works on macOS and Linux (x86_64 and arm64).
+
+### 2. Run a flow (CLI)
+
+```bash
+docling-pipelines --flow-file path/to/flow.json
+```
+
+Validate without executing:
+
+```bash
+docling-pipelines --flow-file flow.json --validate
+```
+
+List all available operators:
+
+```bash
+docling-pipelines --list-operators
+```
+
+### 3. Python API
+
+```python
+from docpipe.lib.docpipe_flow_manager import DocpipeFlowManager
+
+manager = DocpipeFlowManager(flow_file="path/to/flow.json")
+result = manager.execute()
+```
+
+Log verbosity is controlled via `DS_LOG_LEVEL` (`DEBUG`, `INFO`, `WARNING`).
+
+## Documentation
+
+Check out the full [documentation](docs/README.md) for installation, flow authoring, operator reference, and more:
+
+- [Quick Start Guide](QUICKSTART.md) — first pipeline in under 5 minutes
+- [Pipeline Setup Guide](USER_GUIDE_PIPELINE_SETUP.md) — complete setup with Ollama, OpenSearch, and flow examples
+- [Flow Authoring Format](docs/guides/FLOW_AUTHORING_FORMAT.md) — declarative flow authoring
+- [Operator Reference](docs/reference/OPERATORS.md) — full parameter specs for all operators
+- [Architecture](ARCHITECTURE.md) — system design and distributed execution patterns
+- [Troubleshooting](TROUBLESHOOTING.md) — common issues and solutions
+
+## Available Operators
+
+| Category | Operators |
+|---|---|
+| **Ingest** | Local File Ingest (`ingest_local`), Remote Source Ingest (`ingest_source`) — [S3, IBM COS, SharePoint, OneDrive, Google Drive, Box, CSV, web](docs/operators/ingest_source/README.md) |
+| **Extract** | Document Extractor (`extract_operator`), ACL Extraction (`acl_operator`) |
+| **Functional** | Chunking (`chunker`), Embeddings (`embeddings`), Branching Operator (`branching`), Merge Operator (`merge`), Document ID Hash (`doc_id_hash`), Entity Curation (`entity_curation`), No-op (`noop`) |
+| **Quality** | Language Annotator (`lang_detect`), Readability Operator (`readability`), PII and HAP Annotator (`pii_and_hap`), Document Classifier (`document_classifier`), Annotation Filter (`sql_filter`), Redaction (`redaction`), De-duplicator (`ededup`), ML Text Enrichment (`ml_enrichment`), Document Quality (`doc_quality`) |
+| **VectorDB** | Vector Database (`vectordb`) — OpenSearch, Milvus |
+| **Storage** | Document Set (`document_set`) — DuckDB-backed document collections |
+
+For per-operator configuration guides, see [Operator Configuration Guides](docs/reference/OPERATORS.md).
+
+## Examples
+
+Explore [sample flows](examples/) and [DocpipeFlowManager examples](examples/docpipe_flow_manager/) for common pipeline patterns.
+
+For interactive, hands-on tutorials, see the [Jupyter notebook examples](examples/notebooks/README.md).
+
+## Contributing
+
+Please read [Contributing to Docling pipelines](CONTRIBUTING.md) for development setup, code standards, testing requirements, and the pull request process.
+
+## License
+
+The Docling pipelines codebase is under the [MIT License](LICENSE).

{docling_pipelines-1.0.0.dev1 → docling_pipelines-1.0.0.dev2}/pyproject.toml

@@ -2,6 +2,7 @@
 name = "docling-pipelines"
 dynamic = ["version"]
 description = "This repository contains the docpipe operators."
+readme = "README.md"
 requires-python = ">=3.12,<3.13"
 maintainers = [
     { name = "Jojo Joseph", email = "jjojo@in.ibm.com" },

docling_pipelines-1.0.0.dev1/PKG-INFO

@@ -1,113 +0,0 @@
-Metadata-Version: 2.4
-Name: docling-pipelines
-Version: 1.0.0.dev1
-Summary: This repository contains the docpipe operators.
-Maintainer-email: Jojo Joseph <jjojo@in.ibm.com>, Aditya Rohan Singh <adityars@ibm.com>, Anudeep Dunaboyina <anudeep.dunaboyina@ibm.com>, Ashok Kumar Angadi <asangadi@in.ibm.com>, Harshad Nehate <Harshad.Nehate@ibm.com>, Kurian Kevin <Kurian.Kevin@ibm.com>, Nipun Bayas <nipun.bayas@ibm.com>, Paul Baby <Paul.Baby1@ibm.com>, Prasad Madugundu <smadugun@in.ibm.com>, Remya P <remya.p@ibm.com>, Sahil Berwal <Sahil.Berwal@ibm.com>, Sanchit Rote <Sanchit.Rote@ibm.com>, Yash Bopardikar <ybopard@us.ibm.com>, Benson Fernandes <benferna@in.ibm.com>, Manish Soni <Manish.Soni3@ibm.com>, Docling Pipeline Team <docling_pipeline_team-dg@ibm.com>
-License-File: LICENSE
-Requires-Python: <3.13,>=3.12
-Requires-Dist: accelerate==1.13.0
-Requires-Dist: aiohttp==3.14.1
-Requires-Dist: alembic==1.18.4
-Requires-Dist: azure-storage-blob==12.28.0
-Requires-Dist: babel==2.18.0
-Requires-Dist: boto3==1.42.55
-Requires-Dist: botocore==1.42.55
-Requires-Dist: box-sdk-gen==1.17.0
-Requires-Dist: data-prep-toolkit-transforms==1.1.8
-Requires-Dist: datatrove==0.3.0
-Requires-Dist: datefinder==1.0.0
-Requires-Dist: docling[rapidocr]==2.105.0
-Requires-Dist: duckdb==1.3.2
-Requires-Dist: fastapi==0.136.3
-Requires-Dist: fasttext-wheel==0.9.2
-Requires-Dist: filelock==3.24.2
-Requires-Dist: ftfy==6.3.1
-Requires-Dist: google-api-python-client==2.190.0
-Requires-Dist: google-auth-httplib2==0.3.0
-Requires-Dist: google-auth-oauthlib==1.2.4
-Requires-Dist: google-cloud-storage==3.9.0
-Requires-Dist: google-re2==1.1.20251105
-Requires-Dist: griffe==1.15.0
-Requires-Dist: httpx==0.28.1
-Requires-Dist: ibm-cos-sdk==2.14.3
-Requires-Dist: ibm-watsonx-ai==1.5.10
-Requires-Dist: idna==3.15
-Requires-Dist: langchain-classic==1.0.7
-Requires-Dist: langchain-community==0.4.1
-Requires-Dist: langchain-core==1.4.6
-Requires-Dist: langchain-experimental==0.4.1
-Requires-Dist: langchain-text-splitters==1.1.2
-Requires-Dist: langchain==1.3.9
-Requires-Dist: langdetect==1.0.9
-Requires-Dist: langsmith==0.8.4
-Requires-Dist: litellm==1.84.0
-Requires-Dist: lxml==6.1.0
-Requires-Dist: mako==1.3.12
-Requires-Dist: mlx==0.30.6; sys_platform == 'darwin'
-Requires-Dist: msal==1.37.0
-Requires-Dist: nltk==3.10.0
-Requires-Dist: o365==2.1.9
-Requires-Dist: ollama==0.6.1
-Requires-Dist: openai==2.33.0
-Requires-Dist: opensearch-py==3.1.0
-Requires-Dist: orjson==3.11.7
-Requires-Dist: pandas==2.3.3
-Requires-Dist: passlib[bcrypt]==1.7.4
-Requires-Dist: pillow==12.2.0
-Requires-Dist: polars==1.32.3
-Requires-Dist: prefect==3.7.1
-Requires-Dist: psycopg2-binary==2.9.12
-Requires-Dist: pyarrow==24.0.0
-Requires-Dist: pyasn1==0.6.3
-Requires-Dist: pydantic-settings==2.14.0
-Requires-Dist: pydantic==2.12.5
-Requires-Dist: pygments==2.20.0
-Requires-Dist: pyiceberg-core==0.7.0
-Requires-Dist: pyiceberg[glue]==0.9.1
-Requires-Dist: pyjwt==2.13.0
-Requires-Dist: pymilvus==2.6.13
-Requires-Dist: pyopenssl==26.3.0
-Requires-Dist: pypdf==6.13.3
-Requires-Dist: pyphen==0.17.2
-Requires-Dist: python-dotenv==1.2.1
-Requires-Dist: python-jose[cryptography]==3.5.0
-Requires-Dist: python-ldap==3.4.5
-Requires-Dist: pyyaml==6.0.3
-Requires-Dist: requests==2.32.5
-Requires-Dist: sentence-transformers==5.4.1
-Requires-Dist: setuptools==81.0.0
-Requires-Dist: sqlalchemy==2.0.49
-Requires-Dist: sqlglot==27.13.2
-Requires-Dist: sqlmodel==0.0.38
-Requires-Dist: starlette==1.3.1
-Requires-Dist: tabulate==0.9.0
-Requires-Dist: tenacity==8.3.0
-Requires-Dist: toml==0.10.2
-Requires-Dist: torch==2.12.1
-Requires-Dist: transformers==5.8.1
-Requires-Dist: unicategories==0.1.2
-Requires-Dist: unicodedataplus==16.0.0.post1
-Requires-Dist: urllib3==2.7.0
-Requires-Dist: uvicorn[standard]==0.40.0
-Provides-Extra: dev
-Requires-Dist: black==26.1.0; extra == 'dev'
-Requires-Dist: detect-secrets==1.5.0; extra == 'dev'
-Requires-Dist: faker==40.8.0; extra == 'dev'
-Requires-Dist: flake8==7.3.0; extra == 'dev'
-Requires-Dist: httpx==0.28.1; extra == 'dev'
-Requires-Dist: mypy==1.19.1; extra == 'dev'
-Requires-Dist: pre-commit==4.5.1; extra == 'dev'
-Requires-Dist: pytest-asyncio==1.4.0; extra == 'dev'
-Requires-Dist: pytest-cov==7.0.0; extra == 'dev'
-Requires-Dist: pytest-mock==3.15.1; extra == 'dev'
-Requires-Dist: pytest==9.0.2; extra == 'dev'
-Requires-Dist: ruff==0.15.5; extra == 'dev'
-Provides-Extra: notebooks
-Requires-Dist: ipykernel==7.3.0; extra == 'notebooks'
-Requires-Dist: jupyter==1.1.1; extra == 'notebooks'
-Requires-Dist: matplotlib==3.11.0; extra == 'notebooks'
-Requires-Dist: nbconvert==7.17.1; extra == 'notebooks'
-Requires-Dist: nbformat==5.10.4; extra == 'notebooks'
-Requires-Dist: seaborn==0.13.2; extra == 'notebooks'
-Provides-Extra: vlm-asr
-Requires-Dist: docling[asr,vlm]; extra == 'vlm-asr'