docling-pipelines 1.0.0.dev1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1058) hide show
  1. docling_pipelines-1.0.0.dev1/.coveragerc +32 -0
  2. docling_pipelines-1.0.0.dev1/.dockerignore +77 -0
  3. docling_pipelines-1.0.0.dev1/.env.example +122 -0
  4. docling_pipelines-1.0.0.dev1/.env.oauth2.example +123 -0
  5. docling_pipelines-1.0.0.dev1/.gitattributes +4 -0
  6. docling_pipelines-1.0.0.dev1/.github/ISSUE_TEMPLATE/bug_report.md +26 -0
  7. docling_pipelines-1.0.0.dev1/.github/ISSUE_TEMPLATE/config.yml +1 -0
  8. docling_pipelines-1.0.0.dev1/.github/ISSUE_TEMPLATE/feature_request.md +18 -0
  9. docling_pipelines-1.0.0.dev1/.github/ISSUE_TEMPLATE/question.md +14 -0
  10. docling_pipelines-1.0.0.dev1/.github/SECURITY.md +51 -0
  11. docling_pipelines-1.0.0.dev1/.github/dco.yml +2 -0
  12. docling_pipelines-1.0.0.dev1/.github/mergify.yml +22 -0
  13. docling_pipelines-1.0.0.dev1/.github/pull_request_template.md +66 -0
  14. docling_pipelines-1.0.0.dev1/.github/workflows/ci.yml +108 -0
  15. docling_pipelines-1.0.0.dev1/.github/workflows/dco-advisor.yml +185 -0
  16. docling_pipelines-1.0.0.dev1/.github/workflows/pypi.yml +62 -0
  17. docling_pipelines-1.0.0.dev1/.gitignore +77 -0
  18. docling_pipelines-1.0.0.dev1/.pre-commit-config.yaml +50 -0
  19. docling_pipelines-1.0.0.dev1/.secrets.baseline +307 -0
  20. docling_pipelines-1.0.0.dev1/.whitesource +3 -0
  21. docling_pipelines-1.0.0.dev1/AGENTS.md +187 -0
  22. docling_pipelines-1.0.0.dev1/ARCHITECTURE.md +4687 -0
  23. docling_pipelines-1.0.0.dev1/CHANGELOG.md +47 -0
  24. docling_pipelines-1.0.0.dev1/CODEOWNERS +6 -0
  25. docling_pipelines-1.0.0.dev1/COMMUNITY.md +152 -0
  26. docling_pipelines-1.0.0.dev1/CONTRIBUTING.md +1032 -0
  27. docling_pipelines-1.0.0.dev1/LICENSE +21 -0
  28. docling_pipelines-1.0.0.dev1/MAINTAINERS.md +19 -0
  29. docling_pipelines-1.0.0.dev1/PKG-INFO +113 -0
  30. docling_pipelines-1.0.0.dev1/QUICKSTART.md +425 -0
  31. docling_pipelines-1.0.0.dev1/README.md +103 -0
  32. docling_pipelines-1.0.0.dev1/RELEASE_PROCESS.md +248 -0
  33. docling_pipelines-1.0.0.dev1/TROUBLESHOOTING.md +2833 -0
  34. docling_pipelines-1.0.0.dev1/USER_GUIDE_PIPELINE_SETUP.md +968 -0
  35. docling_pipelines-1.0.0.dev1/docker/Dockerfile +50 -0
  36. docling_pipelines-1.0.0.dev1/docker/all-in-one-deployment/.env.example +17 -0
  37. docling_pipelines-1.0.0.dev1/docker/all-in-one-deployment/README.md +172 -0
  38. docling_pipelines-1.0.0.dev1/docker/all-in-one-deployment/docker-compose.yml +157 -0
  39. docling_pipelines-1.0.0.dev1/docker/docker-compose.distributed.yml +265 -0
  40. docling_pipelines-1.0.0.dev1/docker/docker-compose.milvus.yml +131 -0
  41. docling_pipelines-1.0.0.dev1/docker/docker-compose.opensearch.yml +65 -0
  42. docling_pipelines-1.0.0.dev1/docling-pipelines-config.yaml +44 -0
  43. docling_pipelines-1.0.0.dev1/docling-pipelines-config.yaml.example +300 -0
  44. docling_pipelines-1.0.0.dev1/docs/README.md +149 -0
  45. docling_pipelines-1.0.0.dev1/docs/api/ACL_DOCUMENT_RETRIEVAL.md +469 -0
  46. docling_pipelines-1.0.0.dev1/docs/api/OAUTH2_AUTHENTICATION.md +333 -0
  47. docling_pipelines-1.0.0.dev1/docs/api/REST_API_SERVER.md +167 -0
  48. docling_pipelines-1.0.0.dev1/docs/deployment/OPENSHIFT.md +661 -0
  49. docling_pipelines-1.0.0.dev1/docs/guides/ADVANCED_CONFIGURATION.md +315 -0
  50. docling_pipelines-1.0.0.dev1/docs/guides/CREATE_CONNECTOR_GUIDE.md +823 -0
  51. docling_pipelines-1.0.0.dev1/docs/guides/CUSTOM_OPERATORS_GUIDE.md +1346 -0
  52. docling_pipelines-1.0.0.dev1/docs/guides/DEPRECATION_POLICY.md +201 -0
  53. docling_pipelines-1.0.0.dev1/docs/guides/DOCUMENTATION_STYLE_GUIDE.md +403 -0
  54. docling_pipelines-1.0.0.dev1/docs/guides/DOCUMENT_CLASS_UTILS.md +215 -0
  55. docling_pipelines-1.0.0.dev1/docs/guides/EXTERNAL_OPERATOR_INTEGRATION.md +433 -0
  56. docling_pipelines-1.0.0.dev1/docs/guides/FLOW_AUTHORING_FORMAT.md +621 -0
  57. docling_pipelines-1.0.0.dev1/docs/guides/FLOW_CONFIGURATION_GUIDE.md +655 -0
  58. docling_pipelines-1.0.0.dev1/docs/guides/LOGGING_BEST_PRACTICES.md +432 -0
  59. docling_pipelines-1.0.0.dev1/docs/guides/MIGRATION_GUIDE_TEMPLATE.md +139 -0
  60. docling_pipelines-1.0.0.dev1/docs/guides/PYTHON_API_GUIDE.md +589 -0
  61. docling_pipelines-1.0.0.dev1/docs/guides/SECURITY_BEST_PRACTICES.md +300 -0
  62. docling_pipelines-1.0.0.dev1/docs/guides/UNIFIED_LLM_ARCHITECTURE_GUIDE.md +943 -0
  63. docling_pipelines-1.0.0.dev1/docs/guides/USER_GUIDE_DOCUMENT_LIBRARIES.md +604 -0
  64. docling_pipelines-1.0.0.dev1/docs/integrations/milvus/README.md +875 -0
  65. docling_pipelines-1.0.0.dev1/docs/integrations/opensearch/ENVIRONMENT_SETUP.md +46 -0
  66. docling_pipelines-1.0.0.dev1/docs/integrations/opensearch/OPENSEARCH_QUICKSTART.md +20 -0
  67. docling_pipelines-1.0.0.dev1/docs/integrations/opensearch/README.md +20 -0
  68. docling_pipelines-1.0.0.dev1/docs/integrations/opensearch/SCHEMA_TEMPLATES.md +836 -0
  69. docling_pipelines-1.0.0.dev1/docs/integrations/prefect/DISTRIBUTED_EXECUTION_GUIDE.md +1691 -0
  70. docling_pipelines-1.0.0.dev1/docs/internals/DOCUMENT_LIBRARIES_ARCHITECTURE.md +445 -0
  71. docling_pipelines-1.0.0.dev1/docs/internals/NODE_METADATA_AGGREGATION_STRATEGY.md +506 -0
  72. docling_pipelines-1.0.0.dev1/docs/operators/extract/acl_readme.md +223 -0
  73. docling_pipelines-1.0.0.dev1/docs/operators/extract/extract_operator_readme.md +1181 -0
  74. docling_pipelines-1.0.0.dev1/docs/operators/functional/branching_operator_readme.md +128 -0
  75. docling_pipelines-1.0.0.dev1/docs/operators/functional/chunker_readme.md +248 -0
  76. docling_pipelines-1.0.0.dev1/docs/operators/functional/doc_id_hash_readme.md +65 -0
  77. docling_pipelines-1.0.0.dev1/docs/operators/functional/embeddings_readme.md +253 -0
  78. docling_pipelines-1.0.0.dev1/docs/operators/functional/entity_curation_readme.md +217 -0
  79. docling_pipelines-1.0.0.dev1/docs/operators/functional/merge_readme.md +115 -0
  80. docling_pipelines-1.0.0.dev1/docs/operators/functional/noop_readme.md +72 -0
  81. docling_pipelines-1.0.0.dev1/docs/operators/ingest/ingest_local_readme.md +150 -0
  82. docling_pipelines-1.0.0.dev1/docs/operators/ingest/ingest_source_readme.md +790 -0
  83. docling_pipelines-1.0.0.dev1/docs/operators/quality/doc_quality_readme.md +108 -0
  84. docling_pipelines-1.0.0.dev1/docs/operators/quality/document_classifier_readme.md +740 -0
  85. docling_pipelines-1.0.0.dev1/docs/operators/quality/ededup_readme.md +92 -0
  86. docling_pipelines-1.0.0.dev1/docs/operators/quality/language_detection_readme.md +413 -0
  87. docling_pipelines-1.0.0.dev1/docs/operators/quality/ml_enrichment_readme.md +127 -0
  88. docling_pipelines-1.0.0.dev1/docs/operators/quality/pii_and_hap_readme.md +241 -0
  89. docling_pipelines-1.0.0.dev1/docs/operators/quality/readability_readme.md +139 -0
  90. docling_pipelines-1.0.0.dev1/docs/operators/quality/redaction_readme.md +110 -0
  91. docling_pipelines-1.0.0.dev1/docs/operators/quality/sql_filter_readme.md +120 -0
  92. docling_pipelines-1.0.0.dev1/docs/operators/storage/document_set_readme.md +165 -0
  93. docling_pipelines-1.0.0.dev1/docs/operators/vectordb/milvus.md +513 -0
  94. docling_pipelines-1.0.0.dev1/docs/operators/vectordb/opensearch.md +524 -0
  95. docling_pipelines-1.0.0.dev1/docs/operators/vectordb/vectordb_readme.md +255 -0
  96. docling_pipelines-1.0.0.dev1/docs/reference/DOCUMENT_SCHEMAS.md +40 -0
  97. docling_pipelines-1.0.0.dev1/docs/reference/GLOBAL_CONFIG.md +786 -0
  98. docling_pipelines-1.0.0.dev1/docs/reference/OPERATORS.md +2898 -0
  99. docling_pipelines-1.0.0.dev1/examples/LDAP/.env.example +8 -0
  100. docling_pipelines-1.0.0.dev1/examples/LDAP/README.md +296 -0
  101. docling_pipelines-1.0.0.dev1/examples/LDAP/abc_users.ldif +36 -0
  102. docling_pipelines-1.0.0.dev1/examples/LDAP/docker-compose.yml +33 -0
  103. docling_pipelines-1.0.0.dev1/examples/README.md +461 -0
  104. docling_pipelines-1.0.0.dev1/examples/chunker_docling_serve_example.json +57 -0
  105. docling_pipelines-1.0.0.dev1/examples/chunker_example.py +276 -0
  106. docling_pipelines-1.0.0.dev1/examples/connectors/.env.example +51 -0
  107. docling_pipelines-1.0.0.dev1/examples/connectors/README.md +75 -0
  108. docling_pipelines-1.0.0.dev1/examples/connectors/test_google_drive_adapter.py +174 -0
  109. docling_pipelines-1.0.0.dev1/examples/connectors/test_onedrive_adapter.py +158 -0
  110. docling_pipelines-1.0.0.dev1/examples/connectors/test_s3_adapter.py +173 -0
  111. docling_pipelines-1.0.0.dev1/examples/connectors/test_sharepoint_adapter.py +158 -0
  112. docling_pipelines-1.0.0.dev1/examples/connectors/test_web_adapter.py +224 -0
  113. docling_pipelines-1.0.0.dev1/examples/custom_operators/README.md +258 -0
  114. docling_pipelines-1.0.0.dev1/examples/custom_operators/example_custom_operator.py +100 -0
  115. docling_pipelines-1.0.0.dev1/examples/custom_operators/hello_operator.py +51 -0
  116. docling_pipelines-1.0.0.dev1/examples/custom_operators/package_example/README.md +104 -0
  117. docling_pipelines-1.0.0.dev1/examples/custom_operators/package_example/my_custom_operators/__init__.py +6 -0
  118. docling_pipelines-1.0.0.dev1/examples/custom_operators/package_example/my_custom_operators/operators/reverse_operator.py +79 -0
  119. docling_pipelines-1.0.0.dev1/examples/custom_operators/package_example/my_custom_operators/operators/uppercase_operator.py +79 -0
  120. docling_pipelines-1.0.0.dev1/examples/custom_operators/package_example/pyproject.toml +20 -0
  121. docling_pipelines-1.0.0.dev1/examples/deduplication_example.py +61 -0
  122. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/00_complete_example.py +224 -0
  123. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/01_execute_from_file.py +113 -0
  124. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/02_execute_from_dict.py +88 -0
  125. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/03_list_operators.py +51 -0
  126. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/04_custom_configuration.py +108 -0
  127. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/05_notebook_usage.py +105 -0
  128. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/06_basic_test.py +188 -0
  129. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/06_basic_test_flow.json +67 -0
  130. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/README.md +317 -0
  131. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/sample_flow.json +65 -0
  132. docling_pipelines-1.0.0.dev1/examples/docpipe_flow_manager/sample_jupyter_notebook.ipynb +162 -0
  133. docling_pipelines-1.0.0.dev1/examples/document_set_example.py +189 -0
  134. docling_pipelines-1.0.0.dev1/examples/duckdb_job_stats/README.md +205 -0
  135. docling_pipelines-1.0.0.dev1/examples/duckdb_job_stats/duckdb_job_store_example.py +302 -0
  136. docling_pipelines-1.0.0.dev1/examples/embeddings_litellm_example.py +305 -0
  137. docling_pipelines-1.0.0.dev1/examples/embeddings_pipeline_example.py +425 -0
  138. docling_pipelines-1.0.0.dev1/examples/entity_curation_example.py +245 -0
  139. docling_pipelines-1.0.0.dev1/examples/extract_operator_example.py +1013 -0
  140. docling_pipelines-1.0.0.dev1/examples/ingest_local_folder_example.py +52 -0
  141. docling_pipelines-1.0.0.dev1/examples/ingest_source_example.py +96 -0
  142. docling_pipelines-1.0.0.dev1/examples/language_detection_example.py +110 -0
  143. docling_pipelines-1.0.0.dev1/examples/milvus_integration_example.py +518 -0
  144. docling_pipelines-1.0.0.dev1/examples/ml_enrichment_example.py +122 -0
  145. docling_pipelines-1.0.0.dev1/examples/noop_operator_example.py +42 -0
  146. docling_pipelines-1.0.0.dev1/examples/notebooks/.jupyterignore +1 -0
  147. docling_pipelines-1.0.0.dev1/examples/notebooks/01_quickstart.ipynb +448 -0
  148. docling_pipelines-1.0.0.dev1/examples/notebooks/02_operator_showcase.ipynb +792 -0
  149. docling_pipelines-1.0.0.dev1/examples/notebooks/03_document_extraction.ipynb +1117 -0
  150. docling_pipelines-1.0.0.dev1/examples/notebooks/04_embeddings_vectordb.ipynb +920 -0
  151. docling_pipelines-1.0.0.dev1/examples/notebooks/05_quality_operators.ipynb +698 -0
  152. docling_pipelines-1.0.0.dev1/examples/notebooks/06_rag_pipeline.ipynb +1051 -0
  153. docling_pipelines-1.0.0.dev1/examples/notebooks/07_flow_authoring.ipynb +1229 -0
  154. docling_pipelines-1.0.0.dev1/examples/notebooks/README.md +277 -0
  155. docling_pipelines-1.0.0.dev1/examples/opensearch_example_README.md +298 -0
  156. docling_pipelines-1.0.0.dev1/examples/opensearch_integration_example.py +371 -0
  157. docling_pipelines-1.0.0.dev1/examples/operator_metadata_example.py +27 -0
  158. docling_pipelines-1.0.0.dev1/examples/pii_hap_detection_example.py +33 -0
  159. docling_pipelines-1.0.0.dev1/examples/readability_example.py +54 -0
  160. docling_pipelines-1.0.0.dev1/examples/redaction_example.py +60 -0
  161. docling_pipelines-1.0.0.dev1/examples/test_ingest_extract_flow.json +37 -0
  162. docling_pipelines-1.0.0.dev1/pyproject.toml +300 -0
  163. docling_pipelines-1.0.0.dev1/pytest.ini +25 -0
  164. docling_pipelines-1.0.0.dev1/requirements.txt +318 -0
  165. docling_pipelines-1.0.0.dev1/sample_documents/1kb_file.txt +1 -0
  166. docling_pipelines-1.0.0.dev1/sample_documents/hello.txt +6 -0
  167. docling_pipelines-1.0.0.dev1/sample_flows/README.md +435 -0
  168. docling_pipelines-1.0.0.dev1/sample_flows/advanced/branching_dual_embeddings.json +211 -0
  169. docling_pipelines-1.0.0.dev1/sample_flows/advanced/branching_quality_routing.json +342 -0
  170. docling_pipelines-1.0.0.dev1/sample_flows/advanced/hybrid_chunking.json +84 -0
  171. docling_pipelines-1.0.0.dev1/sample_flows/advanced/multi_stage_enrichment.json +180 -0
  172. docling_pipelines-1.0.0.dev1/sample_flows/advanced/quality_branching_merge_pipeline.json +312 -0
  173. docling_pipelines-1.0.0.dev1/sample_flows/advanced/summarization_pipeline.json +58 -0
  174. docling_pipelines-1.0.0.dev1/sample_flows/custom_operators/custom_operators_demo.json +47 -0
  175. docling_pipelines-1.0.0.dev1/sample_flows/custom_operators/hello_operator.json +23 -0
  176. docling_pipelines-1.0.0.dev1/sample_flows/operators/classification_ollama.json +67 -0
  177. docling_pipelines-1.0.0.dev1/sample_flows/operators/entity_curation_ollama.json +82 -0
  178. docling_pipelines-1.0.0.dev1/sample_flows/operators/entity_extraction_litellm.json +68 -0
  179. docling_pipelines-1.0.0.dev1/sample_flows/operators/entity_extraction_watsonx.json +97 -0
  180. docling_pipelines-1.0.0.dev1/sample_flows/operators/pii_hap_detection.json +72 -0
  181. docling_pipelines-1.0.0.dev1/sample_flows/quickstart/basic_ingest_extract.json +35 -0
  182. docling_pipelines-1.0.0.dev1/sample_flows/quickstart/complete_pipeline_ollama.json +144 -0
  183. docling_pipelines-1.0.0.dev1/sample_flows/quickstart/complete_pipeline_watsonx.json +97 -0
  184. docling_pipelines-1.0.0.dev1/sample_flows/use_cases/audio_video_extraction.json +84 -0
  185. docling_pipelines-1.0.0.dev1/sample_flows/use_cases/document_set_management.json +57 -0
  186. docling_pipelines-1.0.0.dev1/sample_flows/use_cases/invoice_processing.json +168 -0
  187. docling_pipelines-1.0.0.dev1/sample_flows/use_cases/s3_to_opensearch.json +130 -0
  188. docling_pipelines-1.0.0.dev1/sample_flows/vectordb/milvus_integration.json +190 -0
  189. docling_pipelines-1.0.0.dev1/sample_flows/vectordb/opensearch_integration.json +175 -0
  190. docling_pipelines-1.0.0.dev1/scripts/check_modified_files.sh +93 -0
  191. docling_pipelines-1.0.0.dev1/scripts/convert_dag_to_authoring.py +308 -0
  192. docling_pipelines-1.0.0.dev1/scripts/deploy-openshift.sh +458 -0
  193. docling_pipelines-1.0.0.dev1/scripts/setup_docling_pipelines_environment.sh +898 -0
  194. docling_pipelines-1.0.0.dev1/scripts/test_examples.py +506 -0
  195. docling_pipelines-1.0.0.dev1/src/docpipe/__init__.py +1 -0
  196. docling_pipelines-1.0.0.dev1/src/docpipe/api/__init__.py +5 -0
  197. docling_pipelines-1.0.0.dev1/src/docpipe/api/api_router.py +27 -0
  198. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/__init__.py +1 -0
  199. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/dependencies.py +158 -0
  200. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/jwt_handler.py +72 -0
  201. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/ldap_auth.py +136 -0
  202. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/models.py +25 -0
  203. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/oauth2_config.py +101 -0
  204. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/oauth2_provider.py +342 -0
  205. docling_pipelines-1.0.0.dev1/src/docpipe/api/auth/oauth2_routes.py +197 -0
  206. docling_pipelines-1.0.0.dev1/src/docpipe/api/dependencies.py +36 -0
  207. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/__init__.py +0 -0
  208. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/authoring_flow_dto.py +448 -0
  209. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/document_dto.py +126 -0
  210. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/document_library_dto.py +468 -0
  211. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/document_library_mapper.py +194 -0
  212. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/document_set_dto.py +540 -0
  213. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/error_dto.py +289 -0
  214. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/field_definitions.py +677 -0
  215. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/flow_dto.py +1063 -0
  216. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/job_run_dto.py +598 -0
  217. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/job_stats_dto.py +263 -0
  218. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/mappers/__init__.py +0 -0
  219. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/mappers/flow_mapper.py +228 -0
  220. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/mappers/job_stats_mapper.py +94 -0
  221. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/mappers/node_stats_mapper.py +95 -0
  222. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/node_stats_dto.py +209 -0
  223. docling_pipelines-1.0.0.dev1/src/docpipe/api/dto/operator_dto.py +288 -0
  224. docling_pipelines-1.0.0.dev1/src/docpipe/api/main.py +419 -0
  225. docling_pipelines-1.0.0.dev1/src/docpipe/api/middleware/__init__.py +9 -0
  226. docling_pipelines-1.0.0.dev1/src/docpipe/api/middleware/api_logging_middleware.py +39 -0
  227. docling_pipelines-1.0.0.dev1/src/docpipe/api/middleware/error_handler.py +282 -0
  228. docling_pipelines-1.0.0.dev1/src/docpipe/api/middleware/payload_validation.py +35 -0
  229. docling_pipelines-1.0.0.dev1/src/docpipe/api/middleware/transaction_middleware.py +110 -0
  230. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/__init__.py +5 -0
  231. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/document_libraries.py +924 -0
  232. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/document_set_utils.py +89 -0
  233. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/document_sets.py +420 -0
  234. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/documents.py +322 -0
  235. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/flows.py +1180 -0
  236. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/job_runs.py +508 -0
  237. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/operators.py +232 -0
  238. docling_pipelines-1.0.0.dev1/src/docpipe/api/routes/validation.py +191 -0
  239. docling_pipelines-1.0.0.dev1/src/docpipe/api/services/__init__.py +0 -0
  240. docling_pipelines-1.0.0.dev1/src/docpipe/api/services/acl_query_builder.py +196 -0
  241. docling_pipelines-1.0.0.dev1/src/docpipe/api/services/opensearch_service.py +136 -0
  242. docling_pipelines-1.0.0.dev1/src/docpipe/cli/__init__.py +1 -0
  243. docling_pipelines-1.0.0.dev1/src/docpipe/cli/docpipe_cli.py +493 -0
  244. docling_pipelines-1.0.0.dev1/src/docpipe/core/README.md +37 -0
  245. docling_pipelines-1.0.0.dev1/src/docpipe/core/__init__.py +1 -0
  246. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/__init__.py +21 -0
  247. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/huggingface/__init__.py +7 -0
  248. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/huggingface/huggingface_adapter.py +105 -0
  249. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/litellm/__init__.py +9 -0
  250. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/litellm/litellm_adapter.py +348 -0
  251. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/llm_adapter_factory.py +299 -0
  252. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/watsonx/__init__.py +9 -0
  253. docling_pipelines-1.0.0.dev1/src/docpipe/core/adapters/watsonx/watsonx_adapter.py +598 -0
  254. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/__init__.py +0 -0
  255. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/__init__.py +10 -0
  256. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/__init__.py +1 -0
  257. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/duckdb/__init__.py +7 -0
  258. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/duckdb/metadata_repository.py +767 -0
  259. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/repositories/__init__.py +1 -0
  260. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/repositories/duckdb_document_library_metadata_repository.py +716 -0
  261. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/storage/__init__.py +1 -0
  262. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/adapters/storage/duckdb_storage.py +221 -0
  263. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/application/__init__.py +1 -0
  264. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/application/services/__init__.py +1 -0
  265. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/application/services/document_library_service.py +678 -0
  266. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/domain/__init__.py +5 -0
  267. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/domain/models/__init__.py +1 -0
  268. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/domain/models/document_library.py +327 -0
  269. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/domain/ports/__init__.py +1 -0
  270. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/domain/ports/document_library_repository.py +295 -0
  271. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/domain/types.py +39 -0
  272. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/factories/__init__.py +7 -0
  273. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_libraries/factories/document_library_repository_factory.py +182 -0
  274. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/__init__.py +0 -0
  275. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/adapters/__init__.py +16 -0
  276. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/adapters/duckdb/__init__.py +17 -0
  277. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/adapters/duckdb/data_store.py +382 -0
  278. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/adapters/duckdb/metadata_repository.py +515 -0
  279. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/adapters/filesystem/__init__.py +1 -0
  280. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/application/__init__.py +0 -0
  281. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/application/services/__init__.py +0 -0
  282. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/application/services/document_set_service.py +741 -0
  283. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/__init__.py +0 -0
  284. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/models/__init__.py +7 -0
  285. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/models/data_card.py +67 -0
  286. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/models/document_set.py +282 -0
  287. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/models/storage_reference.py +57 -0
  288. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/ports/__init__.py +10 -0
  289. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/ports/data_store.py +180 -0
  290. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/ports/metadata_repository.py +220 -0
  291. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/domain/types.py +61 -0
  292. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/factories/__init__.py +23 -0
  293. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/factories/data_store_factory.py +178 -0
  294. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/document_sets/factories/metadata_repository_factory.py +181 -0
  295. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/__init__.py +9 -0
  296. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/__init__.py +3 -0
  297. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/config/__init__.py +5 -0
  298. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/config/repository_factory.py +127 -0
  299. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/repositories/__init__.py +5 -0
  300. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/repositories/flow_filesystem_utils.py +166 -0
  301. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/repositories/local/__init__.py +6 -0
  302. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/repositories/local/file_lock_manager.py +164 -0
  303. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/adapters/repositories/local/local_flow_repository.py +845 -0
  304. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/application/__init__.py +3 -0
  305. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/application/services/__init__.py +6 -0
  306. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/application/services/authoring_compiler.py +284 -0
  307. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/application/services/flow_service.py +770 -0
  308. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/application/services/validation_service.py +411 -0
  309. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/domain/__init__.py +3 -0
  310. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/domain/models/__init__.py +15 -0
  311. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/domain/models/authoring_flow.py +403 -0
  312. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/domain/models/flow.py +176 -0
  313. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/domain/ports/__init__.py +5 -0
  314. docling_pipelines-1.0.0.dev1/src/docpipe/core/assets/flows/domain/ports/flow_repository.py +153 -0
  315. docling_pipelines-1.0.0.dev1/src/docpipe/core/constants/__init__.py +63 -0
  316. docling_pipelines-1.0.0.dev1/src/docpipe/core/constants/constants.py +662 -0
  317. docling_pipelines-1.0.0.dev1/src/docpipe/core/constants/operator_constants.py +1017 -0
  318. docling_pipelines-1.0.0.dev1/src/docpipe/core/data_access/__init__.py +6 -0
  319. docling_pipelines-1.0.0.dev1/src/docpipe/core/data_access/data_access_utils.py +79 -0
  320. docling_pipelines-1.0.0.dev1/src/docpipe/core/data_access/docpipe_data_access_local.py +83 -0
  321. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/__init__.py +3 -0
  322. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/acord_insurance_form.json +619 -0
  323. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/bank_statements.json +279 -0
  324. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/bill_of_lading.json +265 -0
  325. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/business_licenses_permits.json +193 -0
  326. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/claimant_s_statement.json +483 -0
  327. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/client_success_case_study.json +309 -0
  328. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/credit_card_statements.json +363 -0
  329. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/customer_data_table.json +247 -0
  330. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/customerinfo.json +121 -0
  331. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/customs_form.json +249 -0
  332. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/delivery_receipt.json +201 -0
  333. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/diploma_certification.json +183 -0
  334. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/driver_license.json +365 -0
  335. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/expense_reports.json +287 -0
  336. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/federal_law_cs.json +54 -0
  337. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/financial_statement.json +381 -0
  338. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/i_9_form.json +257 -0
  339. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/initial_claimant_s_statement_for_income_replacement_waiver_of_premium_benefits.json +899 -0
  340. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/insurance_claim.json +285 -0
  341. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/invoice.json +886 -0
  342. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/life_insurance_authorization_form.json +175 -0
  343. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/management_quarterly_cs.json +109 -0
  344. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/mortgage_lending_document.json +259 -0
  345. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/national_id_card.json +221 -0
  346. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/order_request_form.json +376 -0
  347. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/passport.json +293 -0
  348. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/patient_intake_form.json +265 -0
  349. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/purchase_order.json +802 -0
  350. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/receipt.json +249 -0
  351. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/remittance_payment_advice.json +275 -0
  352. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/sales_agreements.json +58 -0
  353. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/schooladmissonform.json +173 -0
  354. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/tax_forms_w_9_1099_941_1120.json +229 -0
  355. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/transcripts.json +238 -0
  356. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/utility_bill.json +451 -0
  357. docling_pipelines-1.0.0.dev1/src/docpipe/core/document_classes/w_4_form.json +193 -0
  358. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/__init__.py +33 -0
  359. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/__init__.py +25 -0
  360. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/config/__init__.py +19 -0
  361. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/config/incremental_metadata_factory.py +399 -0
  362. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/stores/__init__.py +13 -0
  363. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/stores/filesystem/__init__.py +9 -0
  364. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/stores/filesystem/filesystem_incremental_store.py +385 -0
  365. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/stores/postgres/__init__.py +11 -0
  366. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/stores/postgres/models.py +29 -0
  367. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/adapters/stores/postgres/postgres_incremental_store.py +260 -0
  368. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/application/__init__.py +12 -0
  369. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/application/services/__init__.py +12 -0
  370. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/application/services/incremental_update_service.py +345 -0
  371. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/domain/__init__.py +13 -0
  372. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/domain/models/__init__.py +11 -0
  373. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/domain/models/incremental_record.py +54 -0
  374. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/domain/ports/__init__.py +11 -0
  375. docling_pipelines-1.0.0.dev1/src/docpipe/core/incremental_metadata/domain/ports/incremental_metadata_store.py +124 -0
  376. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/__init__.py +64 -0
  377. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/__init__.py +35 -0
  378. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/config/__init__.py +22 -0
  379. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/config/job_management_factory.py +545 -0
  380. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/frameworks/__init__.py +19 -0
  381. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/frameworks/default_job_run_manager.py +175 -0
  382. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/services/__init__.py +17 -0
  383. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/services/job_tracker_service.py +1384 -0
  384. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/__init__.py +26 -0
  385. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/duckdb/__init__.py +9 -0
  386. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/duckdb/duckdb_job_stats_store.py +798 -0
  387. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/inmemory/__init__.py +0 -0
  388. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/inmemory/inmemory_job_stats_store.py +455 -0
  389. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/json/__init__.py +0 -0
  390. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/json/json_job_stats_store.py +667 -0
  391. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/__init__.py +10 -0
  392. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/dal/__init__.py +11 -0
  393. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/dal/base_dao.py +267 -0
  394. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/dal/job_stats_dal.py +162 -0
  395. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/dal/node_stats_dal.py +243 -0
  396. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/database.py +208 -0
  397. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/mappers.py +129 -0
  398. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/migrations/alembic.ini +148 -0
  399. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/migrations/env.py +154 -0
  400. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/migrations/script.py.mako +30 -0
  401. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/migrations/versions/20260422_001_initial_schema.py +188 -0
  402. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/models.py +322 -0
  403. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/adapters/stores/postgres/postgres_job_stats_store.py +379 -0
  404. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/__init__.py +13 -0
  405. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/aggregation/__init__.py +16 -0
  406. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/aggregation/aggregator.py +263 -0
  407. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/aggregation/batch_aggregator.py +810 -0
  408. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/aggregation/strategies.py +104 -0
  409. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/services/__init__.py +13 -0
  410. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/services/job_management_service.py +505 -0
  411. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/application/services/node_stats_aggregator.py +139 -0
  412. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/__init__.py +18 -0
  413. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/models/__init__.py +18 -0
  414. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/models/job_stats.py +63 -0
  415. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/models/node_stats.py +79 -0
  416. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/ports/__init__.py +15 -0
  417. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/ports/job_run_manager.py +121 -0
  418. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/ports/job_stats_service.py +551 -0
  419. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/ports/job_stats_store.py +220 -0
  420. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/utils/__init__.py +10 -0
  421. docling_pipelines-1.0.0.dev1/src/docpipe/core/job_management/domain/utils/normalization.py +40 -0
  422. docling_pipelines-1.0.0.dev1/src/docpipe/core/models/__init__.py +0 -0
  423. docling_pipelines-1.0.0.dev1/src/docpipe/core/models/session_info.py +74 -0
  424. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/__init__.py +1 -0
  425. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/abstract_operator.py +116 -0
  426. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/__init__.py +31 -0
  427. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/acl_operator.py +512 -0
  428. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/adapters/__init__.py +15 -0
  429. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/adapters/outbound/__init__.py +17 -0
  430. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/adapters/outbound/factories/__init__.py +14 -0
  431. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/adapters/outbound/factories/acl_adapter_factory.py +165 -0
  432. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/adapters/outbound/sharepoint_adapter.py +812 -0
  433. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/domain/__init__.py +19 -0
  434. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/domain/models.py +145 -0
  435. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/ports/__init__.py +13 -0
  436. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/ports/outbound/__init__.py +13 -0
  437. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/acl/ports/outbound/acl_extraction.py +184 -0
  438. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/application/__init__.py +1 -0
  439. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/application/services/__init__.py +1 -0
  440. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/application/services/operator_metadata_service.py +152 -0
  441. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/document_sets/__init__.py +12 -0
  442. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/document_sets/document_set_operator.py +331 -0
  443. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/__init__.py +5 -0
  444. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/__init__.py +6 -0
  445. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/__init__.py +27 -0
  446. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/entity_extraction/__init__.py +6 -0
  447. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/entity_extraction/docling_entity_adapter.py +459 -0
  448. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/entity_extraction/llm_entity_adapter.py +486 -0
  449. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/factories/__init__.py +13 -0
  450. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/factories/entity_extraction_adapter_factory.py +221 -0
  451. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/factories/text_extraction_adapter_factory.py +384 -0
  452. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/text_extraction/__init__.py +19 -0
  453. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/text_extraction/docling_adapter.py +281 -0
  454. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/adapters/outbound/text_extraction/docling_serve_adapter.py +224 -0
  455. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/domain/__init__.py +19 -0
  456. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/domain/models.py +173 -0
  457. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/extract_operator.py +1095 -0
  458. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/ports/__init__.py +1 -0
  459. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/ports/outbound/__init__.py +6 -0
  460. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/ports/outbound/entity_extraction.py +180 -0
  461. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/ports/outbound/text_extraction.py +524 -0
  462. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/services/__init__.py +0 -0
  463. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/extract/services/entity_extraction_service.py +652 -0
  464. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/__init__.py +17 -0
  465. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/branching_operator.py +357 -0
  466. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/chunker.py +1368 -0
  467. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/chunker_validator.py +300 -0
  468. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/doc_id_hash.py +138 -0
  469. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/__init__.py +37 -0
  470. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/__init__.py +1 -0
  471. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/__init__.py +18 -0
  472. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/factories/__init__.py +1 -0
  473. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/factories/llm_adapter_factory.py +100 -0
  474. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/huggingface_adapter.py +92 -0
  475. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/litellm_adapter.py +166 -0
  476. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/ollama_adapter.py +183 -0
  477. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/adapters/outbound/watsonx_adapter.py +174 -0
  478. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/domain/__init__.py +5 -0
  479. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/domain/models.py +54 -0
  480. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/embeddings_operator.py +1019 -0
  481. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/ports/__init__.py +1 -0
  482. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/ports/outbound/__init__.py +1 -0
  483. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/embeddings/ports/outbound/llm_service.py +90 -0
  484. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/__init__.py +5 -0
  485. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/entity_curation_operator.py +266 -0
  486. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/schema_processor.py +409 -0
  487. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/transforms/__init__.py +29 -0
  488. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/transforms/currency_to_numeric.py +169 -0
  489. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/transforms/make_date_uniform.py +100 -0
  490. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/transforms/to_number.py +178 -0
  491. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/entity_curation/transforms/weight_to_numeric.py +301 -0
  492. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/merge.py +466 -0
  493. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/noop.py +83 -0
  494. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/functional/summarization_service.py +389 -0
  495. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/__init__.py +0 -0
  496. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/__init__.py +1 -0
  497. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/__init__.py +1 -0
  498. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/__init__.py +24 -0
  499. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/box/README.md +200 -0
  500. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/box/__init__.py +10 -0
  501. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/box/adapter.py +308 -0
  502. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/box/config.py +79 -0
  503. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/factories/__init__.py +5 -0
  504. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/factories/source_factory.py +146 -0
  505. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/filesystem/__init__.py +6 -0
  506. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/filesystem/adapter.py +296 -0
  507. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/filesystem/config.py +76 -0
  508. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/filesystem/example_usage.py +138 -0
  509. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/google_drive/README.md +446 -0
  510. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/google_drive/__init__.py +6 -0
  511. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/google_drive/adapter.py +566 -0
  512. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/google_drive/config.py +137 -0
  513. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/onedrive/README.md +260 -0
  514. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/onedrive/__init__.py +6 -0
  515. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/onedrive/adapter.py +367 -0
  516. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/onedrive/config.py +106 -0
  517. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/s3/README.md +488 -0
  518. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/s3/__init__.py +6 -0
  519. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/s3/adapter.py +643 -0
  520. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/s3/config.py +145 -0
  521. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/sharepoint/README.md +296 -0
  522. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/sharepoint/__init__.py +6 -0
  523. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/sharepoint/adapter.py +370 -0
  524. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/sharepoint/config.py +106 -0
  525. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/web/README.md +213 -0
  526. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/web/__init__.py +6 -0
  527. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/web/adapter.py +277 -0
  528. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/adapters/outbound/sources/web/config.py +97 -0
  529. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/domain/__init__.py +1 -0
  530. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/domain/models.py +138 -0
  531. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/ingest_local.py +537 -0
  532. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/ingest_source.py +960 -0
  533. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/ingest_utils.py +50 -0
  534. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/ports/__init__.py +1 -0
  535. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/ports/outbound/__init__.py +1 -0
  536. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/ingest/ports/outbound/document_source.py +153 -0
  537. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/operator_metadata.py +349 -0
  538. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/operator_registry.py +111 -0
  539. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/operator_utils.py +1296 -0
  540. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/__init__.py +1 -0
  541. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/classification/__init__.py +1 -0
  542. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/classification/classification_service.py +291 -0
  543. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/classification/document_classifier.py +859 -0
  544. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/classification/domain/__init__.py +1 -0
  545. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/classification/domain/models.py +103 -0
  546. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/de +66 -0
  547. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/doc_quality.py +149 -0
  548. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/ededup.py +124 -0
  549. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/en +403 -0
  550. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/es +68 -0
  551. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/fr +91 -0
  552. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/ja +180 -0
  553. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/__init__.py +1 -0
  554. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/adapters/__init__.py +1 -0
  555. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/adapters/outbound/__init__.py +14 -0
  556. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/adapters/outbound/factories/__init__.py +1 -0
  557. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/adapters/outbound/factories/language_adapter_factory.py +99 -0
  558. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/adapters/outbound/fasttext_adapter.py +116 -0
  559. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/adapters/outbound/langdetect_adapter.py +86 -0
  560. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/domain/__init__.py +5 -0
  561. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/domain/models.py +21 -0
  562. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/lang_id.py +241 -0
  563. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/ports/__init__.py +1 -0
  564. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/ports/outbound/__init__.py +1 -0
  565. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/language_detection/ports/outbound/language_service.py +52 -0
  566. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/ml_enrichment.py +294 -0
  567. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/__init__.py +1 -0
  568. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/domain/__init__.py +1 -0
  569. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/domain/models.py +67 -0
  570. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/local_pii_hap_detect.py +163 -0
  571. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/pii_and_hap_annotator.py +768 -0
  572. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/pii_and_hap_helper.py +429 -0
  573. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/pii_prompt_examples.json +136 -0
  574. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/services/__init__.py +5 -0
  575. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/pii_and_hap/services/pii_hap_service.py +298 -0
  576. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/readability/__init__.py +8 -0
  577. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/readability/easy_words.txt +2941 -0
  578. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/readability/readability_metrics.py +213 -0
  579. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/readability/readability_operator.py +213 -0
  580. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/redaction.py +193 -0
  581. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/quality/sql_filter.py +690 -0
  582. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/storage/__init__.py +1 -0
  583. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/__init__.py +33 -0
  584. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/__init__.py +1 -0
  585. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/__init__.py +25 -0
  586. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/factories/__init__.py +1 -0
  587. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/factories/vector_store_factory.py +278 -0
  588. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/milvus/__init__.py +21 -0
  589. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/milvus/adapter.py +267 -0
  590. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/milvus/batch_processor.py +317 -0
  591. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/milvus/client.py +262 -0
  592. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/milvus/index_manager.py +544 -0
  593. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/opensearch/__init__.py +13 -0
  594. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/opensearch/adapter.py +241 -0
  595. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/opensearch/batch_processor.py +408 -0
  596. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/opensearch/client.py +230 -0
  597. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/adapters/outbound/opensearch/index_manager.py +1056 -0
  598. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/domain/__init__.py +13 -0
  599. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/domain/models.py +113 -0
  600. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/ports/__init__.py +1 -0
  601. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/ports/outbound/__init__.py +1 -0
  602. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/ports/outbound/vector_store.py +140 -0
  603. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/schemas/default_schema.v1.json +56 -0
  604. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/schemas/full_schema.v1.json +57 -0
  605. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/schemas/template_with_content_analyzer.v1.json +89 -0
  606. docling_pipelines-1.0.0.dev1/src/docpipe/core/operators/vectordb/vectordb_operator.py +686 -0
  607. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/__init__.py +0 -0
  608. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/abstract_flow_execution_event_handler.py +85 -0
  609. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/abstract_operator_executor.py +467 -0
  610. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/abstract_orchestrator.py +663 -0
  611. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/batch_manager.py +287 -0
  612. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/cmdline/__init__.py +0 -0
  613. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/cmdline/cmd_line_operator_executor.py +24 -0
  614. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/feature_propagation/__init__.py +19 -0
  615. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/feature_propagation/features_propagator.py +685 -0
  616. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/feature_propagation/models.py +262 -0
  617. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/flow_execution_event_handler.py +493 -0
  618. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/flow_executor.py +238 -0
  619. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/flow_validator.py +1297 -0
  620. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/futured_list.py +61 -0
  621. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/global_config_metadata.py +196 -0
  622. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_factory.py +257 -0
  623. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/__init__.py +5 -0
  624. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/adapters/__init__.py +10 -0
  625. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/adapters/factories/__init__.py +1 -0
  626. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/adapters/factories/operator_source_factory.py +152 -0
  627. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/adapters/filesystem_adapter.py +240 -0
  628. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/adapters/package_adapter.py +322 -0
  629. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/adapters/s3_adapter.py +195 -0
  630. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/loader_service.py +289 -0
  631. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/ports/__init__.py +1 -0
  632. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/ports/operator_source.py +110 -0
  633. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/operator_loader/validator.py +158 -0
  634. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/orchestrator_factory.py +56 -0
  635. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/__init__.py +19 -0
  636. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/adapters/__init__.py +0 -0
  637. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/adapters/factories/__init__.py +0 -0
  638. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/adapters/factories/batch_execution_factory.py +232 -0
  639. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/adapters/thread_pool_adapter.py +80 -0
  640. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/adapters/work_pool_adapter.py +1115 -0
  641. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/batch_subflow.py +241 -0
  642. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/config/__init__.py +0 -0
  643. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/config/work_pool_config.py +99 -0
  644. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/domain/__init__.py +0 -0
  645. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/domain/models.py +143 -0
  646. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/ports/__init__.py +0 -0
  647. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/ports/batch_execution_port.py +73 -0
  648. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/prefect/prefect_engine.py +753 -0
  649. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/python/__init__.py +0 -0
  650. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/python/python_operator_executor.py +191 -0
  651. docling_pipelines-1.0.0.dev1/src/docpipe/core/orchestration/python/python_orchestrator.py +50 -0
  652. docling_pipelines-1.0.0.dev1/src/docpipe/core/ports/llm_embedding_port.py +89 -0
  653. docling_pipelines-1.0.0.dev1/src/docpipe/core/ports/llm_inference_port.py +79 -0
  654. docling_pipelines-1.0.0.dev1/src/docpipe/core/ports/text_detection_port.py +74 -0
  655. docling_pipelines-1.0.0.dev1/src/docpipe/exceptions/__init__.py +1 -0
  656. docling_pipelines-1.0.0.dev1/src/docpipe/exceptions/docpipe_exceptions.py +707 -0
  657. docling_pipelines-1.0.0.dev1/src/docpipe/exceptions/error_codes.py +99 -0
  658. docling_pipelines-1.0.0.dev1/src/docpipe/exceptions/error_messages.py +127 -0
  659. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/__init__.py +1 -0
  660. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/base_llm_client.py +286 -0
  661. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/docling/__init__.py +1 -0
  662. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/docling/client.py +450 -0
  663. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/docling/vlm_pipeline_options_provider.py +637 -0
  664. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/huggingface/__init__.py +1 -0
  665. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/huggingface/client.py +384 -0
  666. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/litellm/__init__.py +1 -0
  667. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/litellm/client.py +415 -0
  668. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/ollama/__init__.py +1 -0
  669. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/ollama/client.py +915 -0
  670. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/ollama/embeddings.py +69 -0
  671. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/rest_client.py +463 -0
  672. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/watsonx/__init__.py +9 -0
  673. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/watsonx/client.py +324 -0
  674. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/watsonx/model_validator.py +132 -0
  675. docling_pipelines-1.0.0.dev1/src/docpipe/integrations/watsonx/rest_client.py +551 -0
  676. docling_pipelines-1.0.0.dev1/src/docpipe/lib/__init__.py +0 -0
  677. docling_pipelines-1.0.0.dev1/src/docpipe/lib/docpipe_flow_manager.py +453 -0
  678. docling_pipelines-1.0.0.dev1/src/docpipe/storage/__init__.py +36 -0
  679. docling_pipelines-1.0.0.dev1/src/docpipe/storage/duck_db/__init__.py +9 -0
  680. docling_pipelines-1.0.0.dev1/src/docpipe/storage/duck_db/connection.py +6 -0
  681. docling_pipelines-1.0.0.dev1/src/docpipe/storage/duck_db/key_value_storage.py +278 -0
  682. docling_pipelines-1.0.0.dev1/src/docpipe/storage/duck_db/table_storage.py +499 -0
  683. docling_pipelines-1.0.0.dev1/src/docpipe/storage/duck_db/utils.py +52 -0
  684. docling_pipelines-1.0.0.dev1/src/docpipe/storage/exceptions.py +32 -0
  685. docling_pipelines-1.0.0.dev1/src/docpipe/storage/factory.py +86 -0
  686. docling_pipelines-1.0.0.dev1/src/docpipe/storage/file_system/__init__.py +5 -0
  687. docling_pipelines-1.0.0.dev1/src/docpipe/storage/file_system/key_value_storage.py +213 -0
  688. docling_pipelines-1.0.0.dev1/src/docpipe/storage/file_system/utils.py +3 -0
  689. docling_pipelines-1.0.0.dev1/src/docpipe/storage/interfaces/__init__.py +9 -0
  690. docling_pipelines-1.0.0.dev1/src/docpipe/storage/interfaces/key_value_storage.py +113 -0
  691. docling_pipelines-1.0.0.dev1/src/docpipe/storage/interfaces/table_storage.py +130 -0
  692. docling_pipelines-1.0.0.dev1/src/docpipe/utils/__init__.py +1 -0
  693. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/__init__.py +50 -0
  694. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/collections.py +128 -0
  695. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/datetime.py +13 -0
  696. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/docpipe_utils.py +25 -0
  697. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/memmap_file_utils.py +419 -0
  698. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/patterns.py +20 -0
  699. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/strings.py +115 -0
  700. docling_pipelines-1.0.0.dev1/src/docpipe/utils/core/validation.py +441 -0
  701. docling_pipelines-1.0.0.dev1/src/docpipe/utils/data/__init__.py +22 -0
  702. docling_pipelines-1.0.0.dev1/src/docpipe/utils/data/incremental_metadata_store.py +0 -0
  703. docling_pipelines-1.0.0.dev1/src/docpipe/utils/data/pyarrow_handler.py +172 -0
  704. docling_pipelines-1.0.0.dev1/src/docpipe/utils/data/schema_utils.py +93 -0
  705. docling_pipelines-1.0.0.dev1/src/docpipe/utils/data/transform.py +13 -0
  706. docling_pipelines-1.0.0.dev1/src/docpipe/utils/document_class_utils.py +561 -0
  707. docling_pipelines-1.0.0.dev1/src/docpipe/utils/duckdb/__init__.py +5 -0
  708. docling_pipelines-1.0.0.dev1/src/docpipe/utils/duckdb/connection_manager.py +87 -0
  709. docling_pipelines-1.0.0.dev1/src/docpipe/utils/global_config/__init__.py +5 -0
  710. docling_pipelines-1.0.0.dev1/src/docpipe/utils/global_config/display.py +159 -0
  711. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/__init__.py +41 -0
  712. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/caching.py +78 -0
  713. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/concurrency.py +124 -0
  714. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/config.py +226 -0
  715. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/error_formatter.py +277 -0
  716. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/fasttext_model_manager.py +279 -0
  717. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/filesystem.py +55 -0
  718. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/flow_execution_reporter.py +658 -0
  719. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/iam_token_manager.py +229 -0
  720. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/logging.py +333 -0
  721. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/nltk_data_manager.py +204 -0
  722. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/performance.py +137 -0
  723. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/rate_limiter.py +329 -0
  724. docling_pipelines-1.0.0.dev1/src/docpipe/utils/infrastructure/retry.py +104 -0
  725. docling_pipelines-1.0.0.dev1/src/docpipe/utils/operators/__init__.py +18 -0
  726. docling_pipelines-1.0.0.dev1/src/docpipe/utils/operators/binary_content_fetcher.py +291 -0
  727. docling_pipelines-1.0.0.dev1/src/docpipe/utils/operators/config_validation.py +59 -0
  728. docling_pipelines-1.0.0.dev1/src/docpipe/utils/operators/display.py +221 -0
  729. docling_pipelines-1.0.0.dev1/src/docpipe/utils/operators/logging.py +298 -0
  730. docling_pipelines-1.0.0.dev1/src/docpipe/utils/operators/vectordb_utils.py +138 -0
  731. docling_pipelines-1.0.0.dev1/src/docpipe/utils/orchestration/__init__.py +45 -0
  732. docling_pipelines-1.0.0.dev1/src/docpipe/utils/orchestration/deleted_rows_tracker.py +150 -0
  733. docling_pipelines-1.0.0.dev1/src/docpipe/utils/orchestration/elyra_converter.py +1169 -0
  734. docling_pipelines-1.0.0.dev1/src/docpipe/utils/orchestration/flow_utils.py +115 -0
  735. docling_pipelines-1.0.0.dev1/src/docpipe/utils/orchestration/prefect_config.py +151 -0
  736. docling_pipelines-1.0.0.dev1/src/docpipe/utils/summarization_util.py +263 -0
  737. docling_pipelines-1.0.0.dev1/tests/README.md +226 -0
  738. docling_pipelines-1.0.0.dev1/tests/__init__.py +0 -0
  739. docling_pipelines-1.0.0.dev1/tests/conftest.py +345 -0
  740. docling_pipelines-1.0.0.dev1/tests/fixtures/__init__.py +0 -0
  741. docling_pipelines-1.0.0.dev1/tests/fixtures/custom_operators/__init__.py +0 -0
  742. docling_pipelines-1.0.0.dev1/tests/fixtures/custom_operators/invalid_operator_no_short_name.py +24 -0
  743. docling_pipelines-1.0.0.dev1/tests/fixtures/custom_operators/invalid_operator_no_transform.py +19 -0
  744. docling_pipelines-1.0.0.dev1/tests/fixtures/custom_operators/not_an_operator.py +0 -0
  745. docling_pipelines-1.0.0.dev1/tests/fixtures/custom_operators/valid_operator.py +38 -0
  746. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/account_cancel.txt +18 -0
  747. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/cc_update.txt +14 -0
  748. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/feedback.txt +14 -0
  749. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/feedback_duplicate.txt +14 -0
  750. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/free_trial.txt +14 -0
  751. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/order_confirmation.txt +20 -0
  752. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/refund.txt +16 -0
  753. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/special_coupon.txt +20 -0
  754. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/special_occasion.txt +18 -0
  755. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/tech_support.txt +23 -0
  756. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/tech_support_duplicate.txt +23 -0
  757. docling_pipelines-1.0.0.dev1/tests/fixtures/customer_support_docs/travel_expense_tracker.pdf +0 -0
  758. docling_pipelines-1.0.0.dev1/tests/fixtures/invoices/TR-INV_001_3_2.1.pdf +0 -0
  759. docling_pipelines-1.0.0.dev1/tests/fixtures/invoices/TR-INV_001_3_2.1_expected.doclang +313 -0
  760. docling_pipelines-1.0.0.dev1/tests/fixtures/invoices/TR-INV_003_3_2.1.pdf +0 -0
  761. docling_pipelines-1.0.0.dev1/tests/fixtures/invoices/TR-INV_044_1_1.1.pdf +0 -0
  762. docling_pipelines-1.0.0.dev1/tests/fixtures/invoices/invoice.md +34 -0
  763. docling_pipelines-1.0.0.dev1/tests/fixtures/language_docs/English invoice.pdf +0 -0
  764. docling_pipelines-1.0.0.dev1/tests/fixtures/language_docs/Spanish invoice.pdf +0 -0
  765. docling_pipelines-1.0.0.dev1/tests/fixtures/pii_hap_docs/pii_sample.txt +17 -0
  766. docling_pipelines-1.0.0.dev1/tests/fixtures/purchase_orders/PurchaseOrder_PCH-2026-49507.pdf +93 -0
  767. docling_pipelines-1.0.0.dev1/tests/fixtures/purchase_orders/sample_purchase_order.txt +92 -0
  768. docling_pipelines-1.0.0.dev1/tests/fixtures/semantic_chunking_docs/usc04@119-18.pdf +0 -0
  769. docling_pipelines-1.0.0.dev1/tests/integration/__init__.py +0 -0
  770. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/README_ACCURACY_TESTING.md +1066 -0
  771. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/README_NL_TO_SQL_TESTS.md +372 -0
  772. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/document_query_evaluator.py +475 -0
  773. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/document_query_generator.py +485 -0
  774. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/nl_query_generator.py +1317 -0
  775. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/nl_to_sql.py +1195 -0
  776. docling_pipelines-1.0.0.dev1/tests/integration/accuracy_tests/sample_document_generator.py +1504 -0
  777. docling_pipelines-1.0.0.dev1/tests/integration/api/test_document_sets_api.py +515 -0
  778. docling_pipelines-1.0.0.dev1/tests/integration/api/test_documents_api.py +318 -0
  779. docling_pipelines-1.0.0.dev1/tests/integration/app/__init__.py +1 -0
  780. docling_pipelines-1.0.0.dev1/tests/integration/app/routes/__init__.py +1 -0
  781. docling_pipelines-1.0.0.dev1/tests/integration/app/routes/conftest.py +158 -0
  782. docling_pipelines-1.0.0.dev1/tests/integration/app/routes/test_flows_integration.py +811 -0
  783. docling_pipelines-1.0.0.dev1/tests/integration/auth/__init__.py +1 -0
  784. docling_pipelines-1.0.0.dev1/tests/integration/auth/test_jwt_integration.py +188 -0
  785. docling_pipelines-1.0.0.dev1/tests/integration/auth/test_oauth2_flow.py +269 -0
  786. docling_pipelines-1.0.0.dev1/tests/integration/opensearch/test_opensearch_advanced.py +500 -0
  787. docling_pipelines-1.0.0.dev1/tests/integration/opensearch/test_opensearch_connection.py +111 -0
  788. docling_pipelines-1.0.0.dev1/tests/integration/opensearch/test_opensearch_with_container.py +356 -0
  789. docling_pipelines-1.0.0.dev1/tests/integration/operators/__init__.py +0 -0
  790. docling_pipelines-1.0.0.dev1/tests/integration/operators/entity_curation/__init__.py +1 -0
  791. docling_pipelines-1.0.0.dev1/tests/integration/operators/entity_curation/test_entity_curation_integration.py +313 -0
  792. docling_pipelines-1.0.0.dev1/tests/integration/operators/extract/README.md +376 -0
  793. docling_pipelines-1.0.0.dev1/tests/integration/operators/extract/__init__.py +0 -0
  794. docling_pipelines-1.0.0.dev1/tests/integration/operators/extract/adapters/outbound/entity_extraction/test_docling_entity_adapter.py +343 -0
  795. docling_pipelines-1.0.0.dev1/tests/integration/operators/extract/test_extract_docling_serve_integration.py +493 -0
  796. docling_pipelines-1.0.0.dev1/tests/integration/operators/extract/test_extract_operator_integration.py +503 -0
  797. docling_pipelines-1.0.0.dev1/tests/integration/operators/functional/embeddings/test_watsonx_integration.py +363 -0
  798. docling_pipelines-1.0.0.dev1/tests/integration/test_classifier_extract_content_reuse.py +244 -0
  799. docling_pipelines-1.0.0.dev1/tests/integration/test_embeddings_ollama_integration.py +283 -0
  800. docling_pipelines-1.0.0.dev1/tests/integration/test_ingest_extract_chunk_txt.py +243 -0
  801. docling_pipelines-1.0.0.dev1/tests/integration/test_ingest_extract_integration.py +189 -0
  802. docling_pipelines-1.0.0.dev1/tests/integration/test_operator_factory_integration.py +438 -0
  803. docling_pipelines-1.0.0.dev1/tests/integration/test_s3_ingest_extract_pipeline.py +431 -0
  804. docling_pipelines-1.0.0.dev1/tests/manual/test_transaction_logging.py +69 -0
  805. docling_pipelines-1.0.0.dev1/tests/test_docpipe_cli.py +390 -0
  806. docling_pipelines-1.0.0.dev1/tests/test_opensearch_flow_config.py +235 -0
  807. docling_pipelines-1.0.0.dev1/tests/unit/__init__.py +0 -0
  808. docling_pipelines-1.0.0.dev1/tests/unit/api/__init__.py +1 -0
  809. docling_pipelines-1.0.0.dev1/tests/unit/api/dto/__init__.py +0 -0
  810. docling_pipelines-1.0.0.dev1/tests/unit/api/dto/mappers/test_job_stats_mapper.py +310 -0
  811. docling_pipelines-1.0.0.dev1/tests/unit/api/dto/mappers/test_node_stats_mapper.py +354 -0
  812. docling_pipelines-1.0.0.dev1/tests/unit/api/dto/test_error_dto.py +370 -0
  813. docling_pipelines-1.0.0.dev1/tests/unit/api/dto/test_flow_dto.py +718 -0
  814. docling_pipelines-1.0.0.dev1/tests/unit/api/middleware/__init__.py +1 -0
  815. docling_pipelines-1.0.0.dev1/tests/unit/api/middleware/test_exception_handlers.py +341 -0
  816. docling_pipelines-1.0.0.dev1/tests/unit/api/middleware/test_payload_validation.py +317 -0
  817. docling_pipelines-1.0.0.dev1/tests/unit/api/middleware/test_transaction_middleware.py +156 -0
  818. docling_pipelines-1.0.0.dev1/tests/unit/api/routes/__init__.py +0 -0
  819. docling_pipelines-1.0.0.dev1/tests/unit/api/routes/conftest.py +160 -0
  820. docling_pipelines-1.0.0.dev1/tests/unit/api/routes/test_flows_routes.py +1120 -0
  821. docling_pipelines-1.0.0.dev1/tests/unit/api/routes/test_job_runs_routes.py +387 -0
  822. docling_pipelines-1.0.0.dev1/tests/unit/api/routes/test_operators_routes.py +519 -0
  823. docling_pipelines-1.0.0.dev1/tests/unit/api/routes/test_validation_routes.py +467 -0
  824. docling_pipelines-1.0.0.dev1/tests/unit/api/services/test_acl_query_builder.py +218 -0
  825. docling_pipelines-1.0.0.dev1/tests/unit/api/test_app_startup.py +101 -0
  826. docling_pipelines-1.0.0.dev1/tests/unit/api/test_authentication.py +307 -0
  827. docling_pipelines-1.0.0.dev1/tests/unit/api/test_document_library_validation.py +361 -0
  828. docling_pipelines-1.0.0.dev1/tests/unit/api/test_flow_operations.py +146 -0
  829. docling_pipelines-1.0.0.dev1/tests/unit/auth/__init__.py +1 -0
  830. docling_pipelines-1.0.0.dev1/tests/unit/auth/test_jwt_handler.py +202 -0
  831. docling_pipelines-1.0.0.dev1/tests/unit/auth/test_models.py +151 -0
  832. docling_pipelines-1.0.0.dev1/tests/unit/auth/test_oauth2_config.py +185 -0
  833. docling_pipelines-1.0.0.dev1/tests/unit/auth/test_oauth2_provider.py +344 -0
  834. docling_pipelines-1.0.0.dev1/tests/unit/cli/__init__.py +2 -0
  835. docling_pipelines-1.0.0.dev1/tests/unit/cli/test_docpipe_cli.py +472 -0
  836. docling_pipelines-1.0.0.dev1/tests/unit/common/clients/__init__.py +0 -0
  837. docling_pipelines-1.0.0.dev1/tests/unit/common/clients/test_docling_serve_client.py +418 -0
  838. docling_pipelines-1.0.0.dev1/tests/unit/common/clients/test_rest_client.py +833 -0
  839. docling_pipelines-1.0.0.dev1/tests/unit/common/clients/test_vlm_pipeline_options_provider.py +313 -0
  840. docling_pipelines-1.0.0.dev1/tests/unit/common/test_llm_clients.py +334 -0
  841. docling_pipelines-1.0.0.dev1/tests/unit/common/util/core/test_validation.py +645 -0
  842. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_common_utils.py +446 -0
  843. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_config_utils.py +578 -0
  844. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_error_formatter.py +459 -0
  845. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_filesystem_utils.py +367 -0
  846. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_flow_execution_reporter.py +467 -0
  847. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_global_config_display.py +161 -0
  848. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_logging_utils.py +339 -0
  849. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_nltk_data_manager.py +337 -0
  850. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_operator_display_utils.py +270 -0
  851. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_operator_log_details.py +467 -0
  852. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_operator_metadata.py +359 -0
  853. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_orchestrator_utils.py +461 -0
  854. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_performance_utils.py +328 -0
  855. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_pyarrow_handler.py +309 -0
  856. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_retry_utils.py +133 -0
  857. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_schema_utils.py +336 -0
  858. docling_pipelines-1.0.0.dev1/tests/unit/common/util/test_transform_utils.py +257 -0
  859. docling_pipelines-1.0.0.dev1/tests/unit/core/__init__.py +1 -0
  860. docling_pipelines-1.0.0.dev1/tests/unit/core/adapters/__init__.py +1 -0
  861. docling_pipelines-1.0.0.dev1/tests/unit/core/adapters/litellm/__init__.py +1 -0
  862. docling_pipelines-1.0.0.dev1/tests/unit/core/adapters/litellm/test_litellm_adapter.py +320 -0
  863. docling_pipelines-1.0.0.dev1/tests/unit/core/adapters/watsonx/__init__.py +1 -0
  864. docling_pipelines-1.0.0.dev1/tests/unit/core/adapters/watsonx/test_watsonx_adapter.py +487 -0
  865. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/__init__.py +1 -0
  866. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/adapters/__init__.py +1 -0
  867. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/adapters/repositories/__init__.py +1 -0
  868. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/application/__init__.py +1 -0
  869. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/application/services/__init__.py +1 -0
  870. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/application/services/test_document_library_service.py +296 -0
  871. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/conftest.py +115 -0
  872. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/domain/__init__.py +1 -0
  873. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/domain/models/__init__.py +1 -0
  874. docling_pipelines-1.0.0.dev1/tests/unit/core/assets/document_libraries/domain/models/test_document_library_validation.py +224 -0
  875. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/__init__.py +1 -0
  876. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/adapters/__init__.py +0 -0
  877. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/adapters/config/test_repository_factory.py +175 -0
  878. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/adapters/repositories/__init__.py +0 -0
  879. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/adapters/repositories/local/conftest.py +37 -0
  880. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/adapters/repositories/local/test_local_flow_repository.py +1258 -0
  881. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/application/__init__.py +0 -0
  882. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/application/services/__init__.py +0 -0
  883. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/application/services/test_authoring_compiler.py +337 -0
  884. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/application/services/test_flow_service.py +787 -0
  885. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/conftest.py +136 -0
  886. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/domain/__init__.py +0 -0
  887. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/domain/models/__init__.py +0 -0
  888. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/domain/models/test_authoring_flow.py +465 -0
  889. docling_pipelines-1.0.0.dev1/tests/unit/core/assets_management/domain/models/test_flow_validation.py +466 -0
  890. docling_pipelines-1.0.0.dev1/tests/unit/core/flows/application/services/test_validation_service.py +668 -0
  891. docling_pipelines-1.0.0.dev1/tests/unit/core/incremental_metadata/__init__.py +1 -0
  892. docling_pipelines-1.0.0.dev1/tests/unit/core/incremental_metadata/test_factory.py +200 -0
  893. docling_pipelines-1.0.0.dev1/tests/unit/core/incremental_metadata/test_filesystem_store.py +274 -0
  894. docling_pipelines-1.0.0.dev1/tests/unit/core/incremental_metadata/test_incremental_record.py +112 -0
  895. docling_pipelines-1.0.0.dev1/tests/unit/core/incremental_metadata/test_incremental_update_service.py +245 -0
  896. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/__init__.py +0 -0
  897. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/__init__.py +0 -0
  898. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/config/test_job_management_factory.py +337 -0
  899. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/services/test_job_tracker_service.py +2567 -0
  900. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/stores/__init__.py +0 -0
  901. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/stores/postgres/test_migrations.py +253 -0
  902. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/stores/test_duckdb_job_stats_store.py +837 -0
  903. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/stores/test_inmemory_job_stats_store.py +451 -0
  904. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/stores/test_json_job_stats_store.py +916 -0
  905. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/adapters/stores/test_postgres_job_stats_store.py +333 -0
  906. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/application/aggregation/test_metadata_aggregator.py +1126 -0
  907. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/application/services/test_node_stats_aggregator.py +636 -0
  908. docling_pipelines-1.0.0.dev1/tests/unit/core/job_management/domain/utils/test_normalization.py +243 -0
  909. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/__init__.py +1 -0
  910. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/application/__init__.py +0 -0
  911. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/application/services/__init__.py +0 -0
  912. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/application/services/test_operator_metadata_service.py +217 -0
  913. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/document_sets/test_document_set_models.py +427 -0
  914. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/document_sets/test_document_set_operator.py +1 -0
  915. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/document_sets/test_document_set_repository.py +344 -0
  916. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/document_sets/test_document_set_service.py +498 -0
  917. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/document_sets/test_duckdb_adapters.py +496 -0
  918. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/document_sets/test_factories.py +170 -0
  919. docling_pipelines-1.0.0.dev1/tests/unit/core/operators/test_operator_metadata.py +932 -0
  920. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/operator_loader/adapters/test_filesystem_adapter.py +483 -0
  921. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/operator_loader/adapters/test_s3_adapter.py +243 -0
  922. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/operator_loader/test_loader_service.py +574 -0
  923. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/prefect/adapters/test_work_pool_adapter.py +663 -0
  924. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/prefect/test_batch_subflow.py +545 -0
  925. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/prefect/test_prefect_engine_helpers.py +274 -0
  926. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestration/test_flow_execution_event_handler.py +388 -0
  927. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/conftest.py +43 -0
  928. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/operator_loader/__init__.py +0 -0
  929. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/operator_loader/test_package_adapter.py +363 -0
  930. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/operator_loader/test_validator.py +139 -0
  931. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/prefect/test_batch_info_compatibility.py +294 -0
  932. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_abstract_operator_executor_coverage.py +230 -0
  933. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_abstract_operator_executor_empty_docs.py +289 -0
  934. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_abstract_orchestrator.py +404 -0
  935. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_batch_context_executor_integration.py +421 -0
  936. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_batch_manager.py +622 -0
  937. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_cmd_line_operator_executor.py +114 -0
  938. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_cmdline_orchestrator.py +131 -0
  939. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_features_propagator.py +793 -0
  940. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_flow_executor.py +388 -0
  941. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_flow_validator.py +2304 -0
  942. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_global_config_metadata.py +111 -0
  943. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_ingest_source_global_config.py +169 -0
  944. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_operator_factory.py +221 -0
  945. docling_pipelines-1.0.0.dev1/tests/unit/core/orchestrator/test_terminal_path_batch_propagation.py +253 -0
  946. docling_pipelines-1.0.0.dev1/tests/unit/integrations/conftest.py +64 -0
  947. docling_pipelines-1.0.0.dev1/tests/unit/integrations/ollama/test_client.py +437 -0
  948. docling_pipelines-1.0.0.dev1/tests/unit/integrations/watsonx/test_model_validator.py +348 -0
  949. docling_pipelines-1.0.0.dev1/tests/unit/integrations/watsonx/test_rest_client_core.py +369 -0
  950. docling_pipelines-1.0.0.dev1/tests/unit/integrations/watsonx/test_rest_client_token_limits.py +370 -0
  951. docling_pipelines-1.0.0.dev1/tests/unit/operators/acl/__init__.py +1 -0
  952. docling_pipelines-1.0.0.dev1/tests/unit/operators/acl/conftest.py +303 -0
  953. docling_pipelines-1.0.0.dev1/tests/unit/operators/acl/test_acl_adapter_factory.py +344 -0
  954. docling_pipelines-1.0.0.dev1/tests/unit/operators/acl/test_acl_operator.py +505 -0
  955. docling_pipelines-1.0.0.dev1/tests/unit/operators/acl/test_domain_models.py +416 -0
  956. docling_pipelines-1.0.0.dev1/tests/unit/operators/acl/test_sharepoint_adapter.py +618 -0
  957. docling_pipelines-1.0.0.dev1/tests/unit/operators/branching/__init__.py +1 -0
  958. docling_pipelines-1.0.0.dev1/tests/unit/operators/branching/test_branching_operator.py +1044 -0
  959. docling_pipelines-1.0.0.dev1/tests/unit/operators/chunker/__init__.py +1 -0
  960. docling_pipelines-1.0.0.dev1/tests/unit/operators/chunker/conftest.py +14 -0
  961. docling_pipelines-1.0.0.dev1/tests/unit/operators/chunker/test_chunker.py +2011 -0
  962. docling_pipelines-1.0.0.dev1/tests/unit/operators/classify/__init__.py +1 -0
  963. docling_pipelines-1.0.0.dev1/tests/unit/operators/classify/test_classification_domain.py +206 -0
  964. docling_pipelines-1.0.0.dev1/tests/unit/operators/classify/test_document_classifier.py +1490 -0
  965. docling_pipelines-1.0.0.dev1/tests/unit/operators/classify/test_llm_classification_adapter.py +415 -0
  966. docling_pipelines-1.0.0.dev1/tests/unit/operators/doc_id/__init__.py +1 -0
  967. docling_pipelines-1.0.0.dev1/tests/unit/operators/doc_id/test_doc_id_hash.py +497 -0
  968. docling_pipelines-1.0.0.dev1/tests/unit/operators/ededup/test_ededup.py +400 -0
  969. docling_pipelines-1.0.0.dev1/tests/unit/operators/embeddings/__init__.py +3 -0
  970. docling_pipelines-1.0.0.dev1/tests/unit/operators/embeddings/conftest.py +81 -0
  971. docling_pipelines-1.0.0.dev1/tests/unit/operators/embeddings/test_embeddings_operator.py +1059 -0
  972. docling_pipelines-1.0.0.dev1/tests/unit/operators/embeddings/test_ollama_dimension_detection.py +336 -0
  973. docling_pipelines-1.0.0.dev1/tests/unit/operators/entity_curation/__init__.py +2 -0
  974. docling_pipelines-1.0.0.dev1/tests/unit/operators/entity_curation/test_entity_curation_operator.py +178 -0
  975. docling_pipelines-1.0.0.dev1/tests/unit/operators/entity_curation/test_schema_processor.py +428 -0
  976. docling_pipelines-1.0.0.dev1/tests/unit/operators/entity_curation/test_transforms.py +117 -0
  977. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/__init__.py +1 -0
  978. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/adapters/outbound/entity_extraction/__init__.py +1 -0
  979. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/adapters/outbound/entity_extraction/test_llm_entity_adapter.py +437 -0
  980. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/adapters/outbound/text_extraction/__init__.py +0 -0
  981. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/adapters/outbound/text_extraction/test_docling_serve_adapter.py +624 -0
  982. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/ports/test_text_extraction_port.py +422 -0
  983. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/test_extract_operator.py +2177 -0
  984. docling_pipelines-1.0.0.dev1/tests/unit/operators/extract/test_extract_operator_doclang.py +245 -0
  985. docling_pipelines-1.0.0.dev1/tests/unit/operators/filter/__init__.py +1 -0
  986. docling_pipelines-1.0.0.dev1/tests/unit/operators/filter/test_sql_filter.py +1205 -0
  987. docling_pipelines-1.0.0.dev1/tests/unit/operators/functional/embeddings/adapters/test_watsonx_adapter.py +461 -0
  988. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/README.md +140 -0
  989. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/__init__.py +1 -0
  990. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/conftest.py +30 -0
  991. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_box_source_adapter.py +117 -0
  992. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_filesystem_source_adapter.py +267 -0
  993. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_google_drive_source_adapter.py +205 -0
  994. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_ingest_local.py +286 -0
  995. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_ingest_local_table_merge.py +210 -0
  996. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_ingest_source.py +1614 -0
  997. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_ingest_source_validation.py +168 -0
  998. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_microsoft_source_adapters.py +330 -0
  999. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_s3_adapter.py +449 -0
  1000. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_s3_source_adapter.py +571 -0
  1001. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_source_adapter_factory.py +80 -0
  1002. docling_pipelines-1.0.0.dev1/tests/unit/operators/ingest/test_web_source_adapter.py +541 -0
  1003. docling_pipelines-1.0.0.dev1/tests/unit/operators/language/__init__.py +2 -0
  1004. docling_pipelines-1.0.0.dev1/tests/unit/operators/language/conftest.py +53 -0
  1005. docling_pipelines-1.0.0.dev1/tests/unit/operators/language/test_lang_id_fasttext.py +427 -0
  1006. docling_pipelines-1.0.0.dev1/tests/unit/operators/language/test_language_detection.py +480 -0
  1007. docling_pipelines-1.0.0.dev1/tests/unit/operators/merge/__init__.py +1 -0
  1008. docling_pipelines-1.0.0.dev1/tests/unit/operators/merge/test_merge.py +574 -0
  1009. docling_pipelines-1.0.0.dev1/tests/unit/operators/ml_enrichment/__init__.py +3 -0
  1010. docling_pipelines-1.0.0.dev1/tests/unit/operators/ml_enrichment/test_ml_enrichment.py +548 -0
  1011. docling_pipelines-1.0.0.dev1/tests/unit/operators/noop/test_noop.py +640 -0
  1012. docling_pipelines-1.0.0.dev1/tests/unit/operators/pii_and_hap/__init__.py +1 -0
  1013. docling_pipelines-1.0.0.dev1/tests/unit/operators/pii_and_hap/test_domain_models.py +91 -0
  1014. docling_pipelines-1.0.0.dev1/tests/unit/operators/pii_and_hap/test_pii_and_hap_annotator.py +784 -0
  1015. docling_pipelines-1.0.0.dev1/tests/unit/operators/pii_and_hap/test_pii_and_hap_integration.py +233 -0
  1016. docling_pipelines-1.0.0.dev1/tests/unit/operators/pii_and_hap/test_pii_hap_flow.py +67 -0
  1017. docling_pipelines-1.0.0.dev1/tests/unit/operators/pii_and_hap/test_pii_hap_service.py +389 -0
  1018. docling_pipelines-1.0.0.dev1/tests/unit/operators/quality/test_pii_hap_annotator.py +111 -0
  1019. docling_pipelines-1.0.0.dev1/tests/unit/operators/readability/__init__.py +0 -0
  1020. docling_pipelines-1.0.0.dev1/tests/unit/operators/readability/test_readability.py +178 -0
  1021. docling_pipelines-1.0.0.dev1/tests/unit/operators/redaction/__init__.py +2 -0
  1022. docling_pipelines-1.0.0.dev1/tests/unit/operators/redaction/test_redaction.py +211 -0
  1023. docling_pipelines-1.0.0.dev1/tests/unit/operators/storage_operator/__init__.py +1 -0
  1024. docling_pipelines-1.0.0.dev1/tests/unit/operators/storage_operator/test_document_set_operator.py +477 -0
  1025. docling_pipelines-1.0.0.dev1/tests/unit/operators/test_abstract_operator.py +730 -0
  1026. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/__init__.py +0 -0
  1027. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_metadata_aggregation.py +603 -0
  1028. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_metadata_normalization.py +261 -0
  1029. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_milvus_batch_processor.py +414 -0
  1030. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_milvus_client.py +173 -0
  1031. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_milvus_index_manager.py +553 -0
  1032. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_multi_model_embeddings.py +353 -0
  1033. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_opensearch_batch_processor.py +871 -0
  1034. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_opensearch_client.py +451 -0
  1035. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_opensearch_index_manager.py +761 -0
  1036. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_schema_template_loading.py +774 -0
  1037. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_schema_validation.py +383 -0
  1038. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_vectordb_metrics.py +265 -0
  1039. docling_pipelines-1.0.0.dev1/tests/unit/operators/vectordb/test_vectordb_operator.py +582 -0
  1040. docling_pipelines-1.0.0.dev1/tests/unit/storage/__init__.py +1 -0
  1041. docling_pipelines-1.0.0.dev1/tests/unit/storage/duck_db/test_utils.py +122 -0
  1042. docling_pipelines-1.0.0.dev1/tests/unit/storage/test_duckdb_key_value_storage.py +251 -0
  1043. docling_pipelines-1.0.0.dev1/tests/unit/storage/test_duckdb_table_storage.py +540 -0
  1044. docling_pipelines-1.0.0.dev1/tests/unit/storage/test_file_system_storage.py +227 -0
  1045. docling_pipelines-1.0.0.dev1/tests/unit/test_datasift_exceptions.py +350 -0
  1046. docling_pipelines-1.0.0.dev1/tests/unit/test_docpipe_flow_manager.py +1161 -0
  1047. docling_pipelines-1.0.0.dev1/tests/unit/util/__init__.py +1 -0
  1048. docling_pipelines-1.0.0.dev1/tests/unit/util/orchestration/__init__.py +1 -0
  1049. docling_pipelines-1.0.0.dev1/tests/unit/util/orchestration/test_elyra_converter.py +1328 -0
  1050. docling_pipelines-1.0.0.dev1/tests/unit/util/test_lru_cache.py +85 -0
  1051. docling_pipelines-1.0.0.dev1/tests/unit/util/test_memmap_file_utils.py +328 -0
  1052. docling_pipelines-1.0.0.dev1/tests/unit/util/test_operator_utils.py +2019 -0
  1053. docling_pipelines-1.0.0.dev1/tests/unit/utils/infrastructure/test_concurrency.py +354 -0
  1054. docling_pipelines-1.0.0.dev1/tests/unit/utils/infrastructure/test_rate_limiter.py +453 -0
  1055. docling_pipelines-1.0.0.dev1/tests/unit/utils/test_document_class_utils.py +785 -0
  1056. docling_pipelines-1.0.0.dev1/tests/unit/utils/test_fasttext_model_manager.py +199 -0
  1057. docling_pipelines-1.0.0.dev1/tests/unit/utils/test_summarization_util.py +541 -0
  1058. docling_pipelines-1.0.0.dev1/uv.lock +6426 -0
@@ -0,0 +1,32 @@
1
+ [run]
2
+ branch = True
3
+ source = src/docpipe_app
4
+ omit =
5
+ */tests/*
6
+ */test_*.py
7
+ */*_test.py
8
+ */venv/*
9
+ */virtualenv/*
10
+ */.venv/*
11
+ */site-packages/*
12
+ */ui/*
13
+ */examples/*
14
+ */__init__.py
15
+ /data/*
16
+ /sonar/*
17
+ /docs/*
18
+ /dist/*
19
+ /build/*
20
+ /ui/*
21
+
22
+
23
+ [report]
24
+ exclude_lines =
25
+ pragma: no cover
26
+ if __name__ == "__main__":
27
+ def __repr__
28
+ raise AssertionError
29
+ raise NotImplementedError
30
+ if TYPE_CHECKING:
31
+ @abstractmethod
32
+ @abc.abstractmethod
@@ -0,0 +1,77 @@
1
+ # Git
2
+ .git/
3
+ .gitignore
4
+
5
+ # Python
6
+ __pycache__/
7
+ *.py[cod]
8
+ *$py.class
9
+ *.pyc
10
+ *.pyo
11
+ *.pyd
12
+ *.so
13
+ .Python
14
+
15
+ # Virtual environments
16
+ venv/
17
+ .venv/
18
+ env/
19
+ ENV/
20
+ env.bak/
21
+ venv.bak/
22
+
23
+ # uv
24
+ .uv/
25
+
26
+ # IDEs
27
+ .vscode/
28
+ .idea/
29
+ *.swp
30
+ *.swo
31
+ *~
32
+
33
+ # Testing
34
+ .pytest_cache/
35
+ .coverage
36
+ htmlcov/
37
+ *.cover
38
+ .mypy_cache
39
+ .ruff_cache
40
+ pytest.ini
41
+
42
+ # Documentation
43
+ *.md
44
+ docs/
45
+
46
+ # Examples and scripts
47
+ examples/
48
+ scripts/
49
+ scratch/
50
+
51
+ # Data and artifacts
52
+ data/
53
+ artifacts/
54
+
55
+ # Build artifacts
56
+ dist/
57
+ build/
58
+ *.egg-info/
59
+
60
+ # CI/CD
61
+ .github/
62
+ Jenkinsfile
63
+ sonar/
64
+
65
+ # Deployment examples
66
+ k8s-deployment-examples/
67
+
68
+ # Environment
69
+ .env
70
+ .env.local
71
+
72
+ # OS
73
+ .DS_Store
74
+ Thumbs.db
75
+
76
+ # Test directory
77
+ tests/
@@ -0,0 +1,122 @@
1
+ # OpenSearch Connection Configuration
2
+ # Copy this file to .env and update with your actual values
3
+ # ============================================================================
4
+ # PYTHON PATH CONFIGURATION
5
+ # ============================================================================
6
+ # Required for imports to work correctly
7
+ # This should be set relative to the project root
8
+ PYTHONPATH=./src
9
+
10
+
11
+ # ============================================================================
12
+ # LOCAL DOCKER SETUP (using docker-compose.opensearch.yml)
13
+ # ============================================================================
14
+ # To run OpenSearch locally with Docker:
15
+ # docker-compose -f docker-compose.opensearch.yml up -d
16
+ #
17
+ # Use these settings for local Docker instance:
18
+ # OPENSEARCH_HOST=localhost
19
+ # OPENSEARCH_PORT=9200
20
+ # OPENSEARCH_USE_SSL=false
21
+ # OPENSEARCH_VERIFY_CERTS=false
22
+ # OPENSEARCH_USERNAME=admin
23
+ # OPENSEARCH_PASSWORD=<your-opensearch-password>
24
+ #
25
+ # OpenSearch Dashboards will be available at: http://localhost:5601
26
+ # ============================================================================
27
+
28
+ # OpenSearch Server
29
+ OPENSEARCH_HOST=localhost
30
+ OPENSEARCH_PORT=9200
31
+ OPENSEARCH_USE_SSL=false
32
+ OPENSEARCH_VERIFY_CERTS=false
33
+
34
+ # Authentication
35
+ OPENSEARCH_USERNAME=admin
36
+ # OPENSEARCH_PASSWORD=<set-your-password>
37
+
38
+ # AWS Authentication (optional, set to true if using AWS OpenSearch)
39
+ OPENSEARCH_AWS_AUTH=false
40
+ # JWT Authentication (optional, for AWS OpenSearch with JWT tokens)
41
+ # OPENSEARCH_JWT_TOKEN=your-jwt-token-here
42
+ OPENSEARCH_AWS_REGION=us-east-1
43
+
44
+ # Engine Configuration
45
+ OPENSEARCH_ENGINE=faiss
46
+ OPENSEARCH_ALGORITHM=hnsw
47
+ OPENSEARCH_SPACE_TYPE=l2
48
+ OPENSEARCH_VECTOR_DIMENSION=384
49
+
50
+ # Performance Settings
51
+ OPENSEARCH_BATCH_SIZE=100
52
+ OPENSEARCH_CREATE_INDEX=true
53
+
54
+ # Index Configuration
55
+ OPENSEARCH_INDEX_NAME=docpipe_test
56
+ OPENSEARCH_DOC_ID_COLUMN=doc_id_hash
57
+ OPENSEARCH_EMBEDDINGS_COLUMN=embeddings
58
+
59
+ # Document Retrieval API Configuration
60
+ # Default index for document retrieval endpoints
61
+ OPENSEARCH_DEFAULT_INDEX=documents
62
+ # Connection timeout in seconds
63
+ OPENSEARCH_TIMEOUT=30
64
+ # Maximum number of retry attempts
65
+ OPENSEARCH_MAX_RETRIES=3
66
+
67
+ DOCPIPE_INDEX=docpipe_test
68
+ DOCPIPE_MODEL=granite4:latest
69
+
70
+ # Ollama Configuration
71
+ # Ollama server host URL used by OllamaClient
72
+ # Default: http://localhost:11434
73
+ OLLAMA_HOST=http://localhost:11434
74
+
75
+ # ============================================================================
76
+ # VLM ENGINE CREDENTIALS (for ExtractOperator)
77
+ # ============================================================================
78
+ # These credentials are used by extract_operator_example.py for VLM engines
79
+ # that require API access (watsonx, openai, generic)
80
+
81
+ # Watsonx AI (IBM Cloud)
82
+ # Required for --vlm-engine watsonx
83
+ # WATSONX_API_KEY=your_ibm_cloud_api_key_here
84
+ # WATSONX_CONTAINER_KIND=project # optional: project, space, or catalog (default: project)
85
+ # WATSONX_CONTAINER_ID=your_project_or_space_id_here
86
+ # WATSONX_MODEL=meta-llama/llama-3-2-90b-vision-instruct
87
+ # WATSONX_API_BASE_URL=https://us-south.ml.cloud.ibm.com # optional, has default
88
+
89
+ # OpenAI API
90
+ # Required for --vlm-engine openai
91
+ # OPENAI_API_KEY=your_openai_api_key_here
92
+ # OPENAI_MODEL=gpt-4-vision-preview
93
+ # OPENAI_API_BASE_URL=https://api.openai.com/v1 # optional, has default
94
+
95
+ # Generic API (any OpenAI-compatible API)
96
+ # Required for --vlm-engine generic
97
+ # GENERIC_API_BASE_URL=https://your-api.com/v1/chat
98
+ # GENERIC_API_KEY=your_api_key_here # optional, some APIs don't require auth
99
+
100
+ # ============================================================================
101
+ # CORS CONFIGURATION
102
+ # ============================================================================
103
+ # Comma-separated list of allowed origins for CORS
104
+ # For development: http://localhost:3000 (frontend/UI server)
105
+ # For production: https://yourdomain.com,https://app.yourdomain.com
106
+ CORS_ORIGINS=http://localhost:3000
107
+
108
+ # ============================================================================
109
+ # FLOW REPOSITORY CONFIGURATION
110
+ # ============================================================================
111
+ # Type of repository to use for storing flow definitions
112
+ # Options: local (default)
113
+ # If not set, defaults to 'local' repository type
114
+ # FLOW_REPOSITORY_TYPE=local
115
+
116
+ # Local repository settings (when FLOW_REPOSITORY_TYPE=local or not set)
117
+ # Directory where flow JSON files are stored
118
+ # If not set, defaults to:
119
+ # macOS/Linux: ~/Documents/pipeline/assets
120
+ # Windows: %USERPROFILE%\Documents\pipeline\assets
121
+ # Use absolute paths for production deployments
122
+ # LOCAL_FLOWS_DIR=/path/to/custom/flows/directory
@@ -0,0 +1,123 @@
1
+ # OAuth2 and OIDC Configuration Example
2
+ # Copy this file to .env and configure your OAuth2 settings
3
+
4
+ # ============================================================================
5
+ # JWT Configuration (Required for all auth methods)
6
+ # ============================================================================
7
+ JWT_SECRET_KEY=your-secret-key-here-change-this-in-production
8
+ JWT_ALGORITHM=HS256
9
+ JWT_ACCESS_TOKEN_EXPIRE_MINUTES=30
10
+
11
+ # ============================================================================
12
+ # OAuth2 General Settings
13
+ # ============================================================================
14
+ OAUTH2_ENABLED=true
15
+ OAUTH2_PROVIDER=google # Options: google, azure, generic
16
+ OAUTH2_CLIENT_ID=your-client-id
17
+ OAUTH2_CLIENT_SECRET=your-client-secret
18
+ OAUTH2_REDIRECT_URI=http://localhost:8000/auth/oauth2/callback
19
+ OAUTH2_SCOPE=openid profile email
20
+ OAUTH2_TOKEN_VALIDATION=true
21
+ OAUTH2_STATE_SECRET=your-state-secret-here
22
+ OAUTH2_SESSION_EXPIRE_MINUTES=60
23
+
24
+ # ============================================================================
25
+ # Google OAuth2 Configuration
26
+ # ============================================================================
27
+ # When using OAUTH2_PROVIDER=google, these are auto-configured:
28
+ # OAUTH2_DISCOVERY_URL=https://accounts.google.com/.well-known/openid-configuration
29
+ # OAUTH2_AUTHORIZATION_ENDPOINT=https://accounts.google.com/o/oauth2/v2/auth
30
+ # OAUTH2_TOKEN_ENDPOINT=https://oauth2.googleapis.com/token
31
+ # OAUTH2_USERINFO_ENDPOINT=https://openidconnect.googleapis.com/v1/userinfo
32
+ # OAUTH2_JWKS_URI=https://www.googleapis.com/oauth2/v3/certs
33
+ # OIDC_ISSUER=https://accounts.google.com
34
+ # OIDC_AUDIENCE=your-client-id
35
+
36
+ # Google OAuth2 Setup:
37
+ # 1. Go to https://console.cloud.google.com/
38
+ # 2. Create a new project or select existing
39
+ # 3. Enable Google+ API
40
+ # 4. Go to Credentials > Create Credentials > OAuth 2.0 Client ID
41
+ # 5. Set application type to "Web application"
42
+ # 6. Add authorized redirect URI: http://localhost:8000/auth/oauth2/callback
43
+ # 7. Copy Client ID and Client Secret to OAUTH2_CLIENT_ID and OAUTH2_CLIENT_SECRET
44
+
45
+ # ============================================================================
46
+ # Azure AD OAuth2 Configuration
47
+ # ============================================================================
48
+ # When using OAUTH2_PROVIDER=azure, configure:
49
+ AZURE_TENANT_ID=common # Or your specific tenant ID
50
+
51
+ # Azure AD Setup:
52
+ # 1. Go to https://portal.azure.com/
53
+ # 2. Navigate to Azure Active Directory > App registrations
54
+ # 3. Click "New registration"
55
+ # 4. Set redirect URI: http://localhost:8000/auth/oauth2/callback
56
+ # 5. Go to Certificates & secrets > New client secret
57
+ # 6. Copy Application (client) ID to OAUTH2_CLIENT_ID
58
+ # 7. Copy client secret value to OAUTH2_CLIENT_SECRET
59
+ # 8. Copy Directory (tenant) ID to AZURE_TENANT_ID
60
+
61
+ # ============================================================================
62
+ # Generic OIDC Provider Configuration
63
+ # ============================================================================
64
+ # When using OAUTH2_PROVIDER=generic, manually configure all endpoints:
65
+ # OAUTH2_DISCOVERY_URL=https://your-provider.com/.well-known/openid-configuration
66
+ # OAUTH2_AUTHORIZATION_ENDPOINT=https://your-provider.com/oauth2/authorize
67
+ # OAUTH2_TOKEN_ENDPOINT=https://your-provider.com/oauth2/token
68
+ # OAUTH2_USERINFO_ENDPOINT=https://your-provider.com/oauth2/userinfo
69
+ # OAUTH2_JWKS_URI=https://your-provider.com/oauth2/jwks
70
+ # OIDC_ISSUER=https://your-provider.com
71
+ # OIDC_AUDIENCE=your-client-id
72
+
73
+ # Generic OIDC providers include:
74
+ # - Okta: https://your-domain.okta.com/.well-known/openid-configuration
75
+ # - Auth0: https://your-domain.auth0.com/.well-known/openid-configuration
76
+ # - Keycloak: https://your-domain.com/auth/realms/{realm}/.well-known/openid-configuration
77
+ # - GitLab: https://gitlab.com/.well-known/openid-configuration
78
+
79
+ # ============================================================================
80
+ # LDAP Configuration (Optional - for LDAP authentication)
81
+ # ============================================================================
82
+ LDAP_SERVER=ldap://localhost:389
83
+ LDAP_BASE_DN=dc=example,dc=com
84
+ LDAP_USER_DN=ou=users,dc=example,dc=com
85
+ LDAP_BIND_DN=cn=admin,dc=example,dc=com
86
+ # LDAP_BIND_PASSWORD=<set-your-ldap-bind-password>
87
+ LDAP_USE_SSL=false
88
+ LDAP_USE_ACTIVE_DIRECTORY=false
89
+ LDAP_AD_DOMAIN=example.com
90
+
91
+ # ============================================================================
92
+ # CORS Configuration
93
+ # ============================================================================
94
+ CORS_ORIGINS=http://localhost:3000,http://localhost:8080
95
+
96
+ # ============================================================================
97
+ # Usage Examples
98
+ # ============================================================================
99
+
100
+ # Example 1: Google OAuth2
101
+ # OAUTH2_ENABLED=true
102
+ # OAUTH2_PROVIDER=google
103
+ # OAUTH2_CLIENT_ID=123456789-abcdefg.apps.googleusercontent.com
104
+ # OAUTH2_CLIENT_SECRET=GOCSPX-your-secret-here
105
+ # OAUTH2_REDIRECT_URI=http://localhost:8000/auth/oauth2/callback
106
+
107
+ # Example 2: Azure AD OAuth2
108
+ # OAUTH2_ENABLED=true
109
+ # OAUTH2_PROVIDER=azure
110
+ # AZURE_TENANT_ID=your-tenant-id-here
111
+ # OAUTH2_CLIENT_ID=your-application-id-here
112
+ # OAUTH2_CLIENT_SECRET=your-client-secret-here
113
+ # OAUTH2_REDIRECT_URI=http://localhost:8000/auth/oauth2/callback
114
+
115
+ # Example 3: Okta OIDC
116
+ # OAUTH2_ENABLED=true
117
+ # OAUTH2_PROVIDER=generic
118
+ # OAUTH2_DISCOVERY_URL=https://your-domain.okta.com/.well-known/openid-configuration
119
+ # OAUTH2_CLIENT_ID=your-okta-client-id
120
+ # OAUTH2_CLIENT_SECRET=your-okta-client-secret
121
+ # OAUTH2_REDIRECT_URI=http://localhost:8000/auth/oauth2/callback
122
+ # OIDC_ISSUER=https://your-domain.okta.com
123
+ # OIDC_AUDIENCE=your-okta-client-id
@@ -0,0 +1,4 @@
1
+ # Jupyter Notebooks
2
+ *.ipynb filter=nbstripout
3
+ *.ipynb diff=jupyternotebook
4
+ *.ipynb merge=jupyternotebook
@@ -0,0 +1,26 @@
1
+ ---
2
+ name: Bug report
3
+ about: Report an issue to help improve docling-pipelines
4
+ title: ''
5
+ labels: bug
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ ### Bug
11
+ <!-- Describe the buggy behavior you have observed. -->
12
+ ...
13
+
14
+ ### Steps to reproduce
15
+ <!-- Describe the sequence of steps for reproducing the bug. -->
16
+ ...
17
+
18
+ ### docling-pipelines version
19
+ <!-- Copy the output of `docling-pipelines --version`. -->
20
+ ...
21
+
22
+ ### Python version
23
+ <!-- Copy the output of `python --version`. -->
24
+ ...
25
+
26
+ <!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
@@ -0,0 +1 @@
1
+ blank_issues_enabled: false
@@ -0,0 +1,18 @@
1
+ ---
2
+ name: Feature request
3
+ about: Suggest an idea for enhancing docling-pipelines
4
+ title: ''
5
+ labels: enhancement
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ ### Requested feature
11
+ <!-- Describe the feature you have in mind and the user need it addresses. -->
12
+ ...
13
+
14
+ ### Alternatives
15
+ <!-- Describe any alternatives you have considered. -->
16
+ ...
17
+
18
+ <!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
@@ -0,0 +1,14 @@
1
+ ---
2
+ name: Question
3
+ about: Ask a question
4
+ title: ''
5
+ labels: question
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ ### Question
11
+ <!-- Describe what you would like to achieve and which part you need help with. -->
12
+ ...
13
+
14
+ <!-- ⚠️ ATTENTION: When sharing screenshots, attachments, or other data make sure not to include any sensitive information. -->
@@ -0,0 +1,51 @@
1
+ # Security and Disclosure Information Policy for the Docling Project
2
+
3
+ The Docling-pipelines team and community take security bugs seriously. We appreciate your efforts to responsibly disclose your findings, and will make every effort to acknowledge your contributions.
4
+
5
+ ### Security
6
+
7
+ - Participation in the [OpenSSF Best Practices Badge Program](https://bestpractices.coreinfrastructure.org/en/projects/) for Free/Libre and FLOSS projects to ensure that we follow current best practices for quality and security
8
+ - Use of [HTTPS](https://en.wikipedia.org/wiki/HTTPS) for network communication
9
+ - Use of secure protocols for network communication (through the use of HTTPS)
10
+ - Up-to-date support for TLS/SSL (through the use of [OpenSSL](https://www.openssl.org/))
11
+ - Performance of TLS certificate verification by default before sending HTTP headers with private information (through the use of OpenSSL and HTTPS)
12
+ - Distribution of the software via cryptographically signed releases (on the [PyPI](https://pypi.org/), [Quay.io](https://quay.io/organization/docling-project/) and [GHCR.io](https://github.com/orgs/docling-project/packages) package repositories)
13
+ - Use of [GitHub](https://github.com/) Issues for vulnerability reporting and tracking
14
+
15
+ ### Analysis
16
+
17
+ - Use of [Ruff](https://docs.astral.sh/ruff/), [ty](https://docs.astral.sh/ty/) and [Pytest](https://docs.pytest.org/en/7.2.x/) for Python code linting (static and dynamic analysers) on pull requests and builds
18
+ - Use of GitHub Issues for bug reporting and tracking
19
+
20
+ ## Reporting a Vulnerability
21
+
22
+ If you think you've identified a security issue in a Docling project repository, please DO NOT report the issue publicly via the GitHub issue tracker, discussions, or any other public forum.
23
+
24
+ ### Preferred Method: GitHub Private Vulnerability Reporting
25
+
26
+ **We strongly encourage you to use GitHub's Private Vulnerability Reporting feature**, which provides a secure and streamlined process for disclosing security issues:
27
+
28
+ 1. Navigate to the [Security tab](https://github.com/docling-project/docling/security) of the Docling repository
29
+ 2. Click on "Report a vulnerability"
30
+ 3. Fill out the vulnerability report form with as many details as possible
31
+ 4. Submit the report
32
+
33
+ This method allows for:
34
+ - **Secure communication** directly with the maintainers team
35
+ - **Coordinated disclosure** through GitHub's built-in workflow
36
+ - **Automatic tracking** of the vulnerability lifecycle
37
+ - **Credit attribution** when the vulnerability is published
38
+
39
+ ### Alternative Method: Email Disclosure
40
+
41
+ Alternatively, you can send an email with as many details as possible to [docling_pipeline_team-dg@ibm.com](mailto:docling_pipeline_team-dg@ibm.com). This is a distribution list for the maintainers team.
42
+
43
+ **Important:** Please do not create a public issue or discuss the vulnerability in any public channel until it has been addressed.
44
+
45
+ ### Security Vulnerability Response
46
+
47
+ Each report is acknowledged and analyzed by the core maintainers within 3 working days.
48
+
49
+ Any vulnerability information shared with core maintainers stays within the Docling project and will not be disseminated to other projects unless it is necessary to get the issue fixed.
50
+
51
+ After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance.
@@ -0,0 +1,2 @@
1
+ allowRemediationCommits:
2
+ individual: true
@@ -0,0 +1,22 @@
1
+ merge_protections:
2
+ - name: Enforce conventional commit
3
+ description: Make sure that we follow https://www.conventionalcommits.org/en/v1.0.0/
4
+ if:
5
+ - base = main
6
+ success_conditions:
7
+ - "title ~= ^(fix|feat|docs|style|refactor|perf|test|build|ci|chore|revert)(\\(.+\\))?(!)?:"
8
+ - name: Require two reviewers for breaking changes
9
+ description: Breaking changes require additional scrutiny with 2 approvers
10
+ if:
11
+ - base = main
12
+ - "title ~= !:"
13
+ success_conditions:
14
+ - "#approved-reviews-by >= 2"
15
+ - name: Require one reviewer for small changes
16
+ description: All non-breaking PRs to main require at least 1 approval
17
+ if:
18
+ - base = main
19
+ success_conditions:
20
+ - "#approved-reviews-by >= 1"
21
+ merge_protections_settings:
22
+ reporting_method: check-runs
@@ -0,0 +1,66 @@
1
+ <!--
2
+ Note -
3
+ 1. PR title must follow Conventional Commits: https://www.conventionalcommits.org/en/v1.0.0/#summary
4
+ Format: <type>[optional scope][optional !]: <description>
5
+ Types: fix | feat | docs | style | refactor | perf | test | build | ci | chore | revert
6
+ Example: feat(chunker): add sliding window chunking strategy
7
+ Append ! before the colon for breaking changes: feat(api)!: remove deprecated endpoint
8
+ 2. Tag the issue this PR resolves (or remove the Issue section if there is none)
9
+ 3. Headings that are not applicable to the PR can be removed from the below template
10
+ -->
11
+
12
+ ## Issue
13
+ <!-- Tag the issue this PR resolves, or remove this section if there is none -->
14
+ Resolves #
15
+
16
+ ## Dev Tracking
17
+ <!-- Link to epic or parent task if this PR is part of a larger feature/epic -->
18
+ <!-- Example: Part of Epic #123 or Subtask of #456 -->
19
+
20
+ ## Description
21
+ <!-- Concise Summary of the PR -->
22
+
23
+ ## Basic Checklist
24
+ - [ ] Code follows project standards (keyword-only arguments, no unicode in logging)
25
+ - [ ] Tests added/updated for new functionality
26
+ - [ ] Documentation updated (see Documentation Update Checklist below)
27
+ - [ ] No breaking changes (or documented in Breaking Changes section)
28
+
29
+ ## Breaking Changes
30
+ <!-- List any breaking changes introduced by this PR -->
31
+ <!-- If none, write "None" -->
32
+
33
+ ## Dependencies
34
+ <!-- List any new dependencies added or version updates -->
35
+ <!-- Include package name, version, and reason for addition/update -->
36
+ <!-- If none, write "None" -->
37
+
38
+ ## Testing Details
39
+ <!-- Details about how the changes were tested -->
40
+ <!-- Include test commands, test files, or manual testing steps -->
41
+
42
+ ## Documentation Update Checklist
43
+ <!-- Check all documentation files that were updated or reviewed for this PR -->
44
+ - [ ] README.md (if adding features, changing structure, or modifying setup)
45
+ - [ ] ARCHITECTURE.md (if modifying operators, data flow, or system design)
46
+ - [ ] CONTRIBUTING.md (if changing development workflow or code standards)
47
+ - [ ] USER_GUIDE_PIPELINE_SETUP.md (if changing installation, setup, or execution)
48
+ - [ ] QUICKSTART.md (if changing quick start steps or examples)
49
+ - [ ] OPERATOR_REFERENCE.md (if adding/modifying operators)
50
+ - [ ] TROUBLESHOOTING.md (if discovering new issues or solutions)
51
+ - [ ] Operator documentation in docs/operators/ (if operator added/modified)
52
+ - [ ] N/A - No documentation updates needed
53
+
54
+ ## Screenshots/Videos
55
+ <!-- For UI changes, include screenshots or videos demonstrating the changes -->
56
+ <!-- If not applicable, remove this section -->
57
+
58
+ ### Pre-Commit Hook Results
59
+ <!-- Paste the output of pre-commit hooks execution to verify all checks passed -->
60
+
61
+
62
+ ### Note
63
+ 1. **Keyword Arguments** - Ensure function arguments are keyword-based, not positional - [Link](https://docs.python.org/3/glossary.html#term-argument)
64
+ 2. **Metadata Independence:** When adding or modifying operator metadata, ensure the data does not depend on instance variables.
65
+ 3. **Method Requirements:** Ensure `get_metadata` is a static method. If `get_required_features` depends on instance variables, the operator must implement `get_static_required_features` using the default values referenced in `get_required_features`
66
+ 4. For every PR, ensure that the pre-commit hook output has been included.
@@ -0,0 +1,108 @@
1
+ name: CI Pipeline
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+ branches: [main]
8
+ workflow_dispatch:
9
+
10
+ concurrency:
11
+ group: ${{ github.workflow }}-${{ github.ref }}
12
+ cancel-in-progress: true
13
+
14
+ jobs:
15
+ lint:
16
+ name: Code Quality & Type Checking
17
+ runs-on: ubuntu-latest
18
+
19
+ steps:
20
+ - name: Checkout Repository
21
+ uses: actions/checkout@v4
22
+ with:
23
+ fetch-depth: 0
24
+
25
+ - name: Install uv
26
+ uses: astral-sh/setup-uv@v5
27
+ with:
28
+ version: "latest"
29
+ enable-cache: true
30
+ cache-suffix: "py312"
31
+
32
+ - name: Set up Python 3.12
33
+ uses: actions/setup-python@v5
34
+ with:
35
+ python-version: "3.12"
36
+
37
+ - name: Install System Dependencies
38
+ run: sudo apt-get install -y libldap2-dev libsasl2-dev
39
+
40
+ - name: Install Dependencies
41
+ run: uv sync --extra dev
42
+
43
+ - name: Check Formatting (Ruff)
44
+ run: |
45
+ MODIFIED=$(git diff --name-only --diff-filter=ACMR $(git merge-base HEAD origin/main)..HEAD | grep '\.py$' || true)
46
+ if [ -z "$MODIFIED" ]; then echo "No modified Python files."; exit 0; fi
47
+ uv run ruff format --check $MODIFIED
48
+
49
+ - name: Lint Code (Ruff)
50
+ run: |
51
+ MODIFIED=$(git diff --name-only --diff-filter=ACMR $(git merge-base HEAD origin/main)..HEAD | grep '\.py$' || true)
52
+ if [ -z "$MODIFIED" ]; then echo "No modified Python files."; exit 0; fi
53
+ uv run ruff check $MODIFIED
54
+
55
+ - name: Static Type Checking (Mypy)
56
+ run: |
57
+ MODIFIED=$(git diff --name-only --diff-filter=ACMR $(git merge-base HEAD origin/main)..HEAD | grep '\.py$' | grep -v '^examples/' || true)
58
+ if [ -z "$MODIFIED" ]; then echo "No modified Python files."; exit 0; fi
59
+ uv run mypy --config-file=pyproject.toml $MODIFIED
60
+
61
+ test:
62
+ name: Test Suite (Python ${{ matrix.python-version }})
63
+ runs-on: ubuntu-latest
64
+ needs: lint
65
+
66
+ strategy:
67
+ fail-fast: false
68
+ matrix:
69
+ python-version: ["3.12"]
70
+
71
+ env:
72
+ PYTHONPATH: src
73
+
74
+ steps:
75
+ - name: Checkout Repository
76
+ uses: actions/checkout@v4
77
+
78
+ - name: Install uv
79
+ uses: astral-sh/setup-uv@v5
80
+ with:
81
+ version: "latest"
82
+ enable-cache: true
83
+ cache-suffix: "py312"
84
+
85
+ - name: Set up Python ${{ matrix.python-version }}
86
+ uses: actions/setup-python@v5
87
+ with:
88
+ python-version: ${{ matrix.python-version }}
89
+
90
+ - name: Install System Dependencies
91
+ run: sudo apt-get install -y libldap2-dev libsasl2-dev
92
+
93
+ - name: Install Dependencies
94
+ run: uv sync --extra dev
95
+
96
+ - name: Run Unit Tests
97
+ run: uv run pytest tests/unit -m "unit and not integration" -v --cov=src --cov-report=xml --cov-report=term-missing
98
+
99
+ - name: Run CLI Tests
100
+ run: uv run pytest tests/test_docpipe_cli.py -v
101
+
102
+ - name: Upload Test Coverage Artifact
103
+ uses: actions/upload-artifact@v4
104
+ if: always()
105
+ with:
106
+ name: coverage-report
107
+ path: coverage.xml
108
+ retention-days: 14