qdrant-loader 1.0.2__tar.gz → 1.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (271) hide show
  1. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/PKG-INFO +20 -7
  2. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/README.md +2 -2
  3. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/pyproject.toml +23 -5
  4. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/__init__.py +8 -1
  5. qdrant_loader-1.0.4/src/qdrant_loader/cli/__main__.py +6 -0
  6. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/cli.py +48 -6
  7. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/__init__.py +15 -0
  8. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/config.py +3 -1
  9. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/config_cmd.py +3 -1
  10. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/ingest.py +2 -0
  11. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/ingest_cmd.py +6 -2
  12. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/init_cmd.py +1 -1
  13. qdrant_loader-1.0.4/src/qdrant_loader/cli/commands/jobs_cmd.py +259 -0
  14. qdrant_loader-1.0.4/src/qdrant_loader/cli/commands/serve_cmd.py +339 -0
  15. qdrant_loader-1.0.4/src/qdrant_loader/cli/commands/webhook_cmd.py +76 -0
  16. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/logging_utils.py +1 -1
  17. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/__init__.py +45 -0
  18. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/chunking.py +33 -0
  19. qdrant_loader-1.0.4/src/qdrant_loader/config/concurrency.py +36 -0
  20. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/embedding.py +10 -0
  21. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/global_config.py +21 -0
  22. qdrant_loader-1.0.4/src/qdrant_loader/config/graph.py +63 -0
  23. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/state.py +19 -0
  24. qdrant_loader-1.0.4/src/qdrant_loader/config/workers.py +202 -0
  25. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/workspace.py +1 -1
  26. qdrant_loader-1.0.4/src/qdrant_loader/connectors/base.py +113 -0
  27. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/connector.py +237 -137
  28. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/pagination.py +20 -4
  29. qdrant_loader-1.0.4/src/qdrant_loader/connectors/factory.py +27 -0
  30. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/connector.py +47 -11
  31. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/cloud_connector.py +30 -9
  32. qdrant_loader-1.0.4/src/qdrant_loader/connectors/jira/config.py +317 -0
  33. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/connector.py +160 -69
  34. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/data_center_connector.py +32 -9
  35. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/mappers.py +94 -13
  36. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/models.py +27 -1
  37. qdrant_loader-1.0.4/src/qdrant_loader/connectors/localfile/connector.py +263 -0
  38. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/connector.py +69 -32
  39. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/async_ingestion_pipeline.py +61 -34
  40. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/chunking_service.py +12 -0
  41. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/__init__.py +48 -0
  42. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/chunker.py +72 -0
  43. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/config.py +127 -0
  44. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/docling_chunker.py +155 -0
  45. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/exceptions.py +26 -0
  46. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/mapper.py +111 -0
  47. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/outcome.py +27 -0
  48. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/structure.py +117 -0
  49. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/tokenizer.py +81 -0
  50. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/__init__.py +4 -0
  51. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base_strategy.py +8 -1
  52. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/strategy/docling_strategy.py +156 -0
  53. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/chunk_processor.py +10 -61
  54. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/section_splitter.py +14 -8
  55. qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/strategy/markdown/splitters/row_kv_excel.py +213 -0
  56. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/__init__.py +39 -0
  57. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/config.py +126 -0
  58. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/docling_engine.py +67 -0
  59. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/engine.py +56 -0
  60. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/formats.py +54 -0
  61. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/options.py +141 -0
  62. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/outcome.py +100 -0
  63. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/postprocess.py +131 -0
  64. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/service.py +257 -0
  65. qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/settings.py +101 -0
  66. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/document.py +32 -3
  67. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/embedding/embedding_service.py +18 -2
  68. qdrant_loader-1.0.4/src/qdrant_loader/core/file_conversion/clean_xlsx_converter.py +164 -0
  69. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/conversion_config.py +17 -0
  70. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/file_converter.py +15 -6
  71. qdrant_loader-1.0.4/src/qdrant_loader/core/file_conversion/sub_table_detector.py +110 -0
  72. qdrant_loader-1.0.4/src/qdrant_loader/core/file_conversion/xlsx_markdown_format.py +23 -0
  73. qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/document_pipeline.py +356 -0
  74. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/factory.py +1 -0
  75. qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/orchestrator.py +867 -0
  76. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/source_processor.py +87 -4
  77. qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/workers/embedding_worker.py +224 -0
  78. qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/workers/upsert_worker.py +443 -0
  79. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/project_manager.py +33 -4
  80. qdrant_loader-1.0.4/src/qdrant_loader/core/qdrant_manager.py +568 -0
  81. qdrant_loader-1.0.4/src/qdrant_loader/core/state/checkpoint_manager.py +286 -0
  82. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/document_state_manager.py +1 -0
  83. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/models.py +53 -4
  84. qdrant_loader-1.0.4/src/qdrant_loader/core/state/session.py +73 -0
  85. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/state_change_detector.py +80 -0
  86. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/state_manager.py +176 -1
  87. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/transitions.py +203 -100
  88. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/utils.py +32 -0
  89. qdrant_loader-1.0.4/src/qdrant_loader/core/text_processing/chunk_enricher.py +91 -0
  90. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/text_processing/semantic_analyzer.py +170 -88
  91. qdrant_loader-1.0.4/src/qdrant_loader/core/text_processing/spacy_model_cache.py +51 -0
  92. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/text_processing/text_processor.py +30 -24
  93. qdrant_loader-1.0.4/src/qdrant_loader/core/worker/__init__.py +30 -0
  94. qdrant_loader-1.0.4/src/qdrant_loader/core/worker/handlers.py +215 -0
  95. qdrant_loader-1.0.4/src/qdrant_loader/core/worker/job_types.py +29 -0
  96. qdrant_loader-1.0.4/src/qdrant_loader/core/worker/pool.py +381 -0
  97. qdrant_loader-1.0.4/src/qdrant_loader/core/worker/queue.py +442 -0
  98. qdrant_loader-1.0.4/src/qdrant_loader/core/worker/scheduler.py +183 -0
  99. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/logging.py +74 -0
  100. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/__init__.py +5 -0
  101. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/auth.py +273 -0
  102. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/event_processor.py +216 -0
  103. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/handlers.py +194 -0
  104. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/queue_backend.py +174 -0
  105. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/server.py +327 -0
  106. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/single_event_handler.py +67 -0
  107. qdrant_loader-1.0.4/src/qdrant_loader/webhooks/worker.py +86 -0
  108. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/PKG-INFO +20 -7
  109. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/SOURCES.txt +49 -1
  110. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/requires.txt +23 -3
  111. qdrant_loader-1.0.2/src/qdrant_loader/connectors/base.py +0 -48
  112. qdrant_loader-1.0.2/src/qdrant_loader/connectors/factory.py +0 -15
  113. qdrant_loader-1.0.2/src/qdrant_loader/connectors/jira/config.py +0 -151
  114. qdrant_loader-1.0.2/src/qdrant_loader/connectors/localfile/connector.py +0 -206
  115. qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/document_pipeline.py +0 -94
  116. qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/orchestrator.py +0 -400
  117. qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/workers/embedding_worker.py +0 -158
  118. qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/workers/upsert_worker.py +0 -287
  119. qdrant_loader-1.0.2/src/qdrant_loader/core/qdrant_manager.py +0 -384
  120. qdrant_loader-1.0.2/src/qdrant_loader/core/state/session.py +0 -37
  121. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/setup.cfg +0 -0
  122. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/__init__.py +0 -0
  123. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/async_utils.py +0 -0
  124. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/asyncio.py +0 -0
  125. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/init.py +0 -0
  126. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/__init__.py +0 -0
  127. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/list_cmd.py +0 -0
  128. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/status_cmd.py +0 -0
  129. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/validate_cmd.py +0 -0
  130. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/setup_cmd.py +0 -0
  131. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/config_loader.py +0 -0
  132. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/path_utils.py +0 -0
  133. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/project_commands.py +0 -0
  134. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/update_check.py +0 -0
  135. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/version.py +0 -0
  136. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/base.py +0 -0
  137. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/error_formatter.py +0 -0
  138. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/models.py +0 -0
  139. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/parser.py +0 -0
  140. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/qdrant.py +0 -0
  141. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/source_config.py +0 -0
  142. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/sources.py +0 -0
  143. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/types.py +0 -0
  144. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/validator.py +0 -0
  145. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config.py +0 -0
  146. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/__init__.py +0 -0
  147. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/attachments.py +0 -0
  148. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/__init__.py +0 -0
  149. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/auth.py +0 -0
  150. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/config.py +0 -0
  151. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/mappers.py +0 -0
  152. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/exceptions.py +0 -0
  153. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/__init__.py +0 -0
  154. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/adapter.py +0 -0
  155. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/config.py +0 -0
  156. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/file_processor.py +0 -0
  157. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/metadata_extractor.py +0 -0
  158. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/operations.py +0 -0
  159. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/http.py +0 -0
  160. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/__init__.py +0 -0
  161. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/auth.py +0 -0
  162. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/__init__.py +0 -0
  163. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/config.py +0 -0
  164. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/file_processor.py +0 -0
  165. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/metadata_extractor.py +0 -0
  166. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/__init__.py +0 -0
  167. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/config.py +0 -0
  168. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/crawler.py +0 -0
  169. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/http.py +0 -0
  170. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/parsers.py +0 -0
  171. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/registry.py +0 -0
  172. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/__init__.py +0 -0
  173. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/attachments/__init__.py +0 -0
  174. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/attachments/metadata.py +0 -0
  175. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/attachments/reader.py +0 -0
  176. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/__init__.py +0 -0
  177. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/client.py +0 -0
  178. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/errors.py +0 -0
  179. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/policy.py +0 -0
  180. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/rate_limit.py +0 -0
  181. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/__init__.py +0 -0
  182. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/attachment_downloader.py +0 -0
  183. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/__init__.py +0 -0
  184. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/progress_tracker.py +0 -0
  185. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/__init__.py +0 -0
  186. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/chunk_processor.py +0 -0
  187. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/document_parser.py +0 -0
  188. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/metadata_extractor.py +0 -0
  189. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/section_splitter.py +0 -0
  190. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/__init__.py +0 -0
  191. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_chunk_processor.py +0 -0
  192. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_document_parser.py +0 -0
  193. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_metadata_extractor.py +0 -0
  194. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_section_splitter.py +0 -0
  195. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/__init__.py +0 -0
  196. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/complexity.py +0 -0
  197. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/dependencies.py +0 -0
  198. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/documentation.py +0 -0
  199. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/entities.py +0 -0
  200. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/language_specific.py +0 -0
  201. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/maintainability.py +0 -0
  202. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/patterns.py +0 -0
  203. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/performance.py +0 -0
  204. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/security.py +0 -0
  205. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/testing.py +0 -0
  206. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/__init__.py +0 -0
  207. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/common.py +0 -0
  208. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/python_ast.py +0 -0
  209. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/tree_sitter.py +0 -0
  210. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/__init__.py +0 -0
  211. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/analysis.py +0 -0
  212. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/quality.py +0 -0
  213. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/utils.py +0 -0
  214. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code_strategy.py +0 -0
  215. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/__init__.py +0 -0
  216. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_chunk_processor.py +0 -0
  217. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_document_parser.py +0 -0
  218. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_metadata_extractor.py +0 -0
  219. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_section_splitter.py +0 -0
  220. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default_strategy.py +0 -0
  221. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/__init__.py +0 -0
  222. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_chunk_processor.py +0 -0
  223. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_document_parser.py +0 -0
  224. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_metadata_extractor.py +0 -0
  225. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_section_splitter.py +0 -0
  226. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html_strategy.py +0 -0
  227. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/__init__.py +0 -0
  228. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_chunk_processor.py +0 -0
  229. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_document_parser.py +0 -0
  230. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_metadata_extractor.py +0 -0
  231. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_section_splitter.py +0 -0
  232. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json_strategy.py +0 -0
  233. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/__init__.py +0 -0
  234. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/document_parser.py +0 -0
  235. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/markdown_strategy.py +0 -0
  236. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/metadata_extractor.py +0 -0
  237. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/__init__.py +0 -0
  238. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/base.py +0 -0
  239. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/excel.py +0 -0
  240. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/fallback.py +0 -0
  241. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/standard.py +0 -0
  242. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/embedding/__init__.py +0 -0
  243. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/embedding_service.py +0 -0
  244. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/__init__.py +0 -0
  245. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/exceptions.py +0 -0
  246. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/file_detector.py +0 -0
  247. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/init_collection.py +0 -0
  248. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/__init__.py +0 -0
  249. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/batch_summary.py +0 -0
  250. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/ingestion_metrics.py +0 -0
  251. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/processing_stats.py +0 -0
  252. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/prometheus_metrics.py +0 -0
  253. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/resource_monitor.py +0 -0
  254. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/__init__.py +0 -0
  255. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/config.py +0 -0
  256. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/resource_manager.py +0 -0
  257. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/source_filter.py +0 -0
  258. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/workers/__init__.py +0 -0
  259. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/workers/base_worker.py +0 -0
  260. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/workers/chunking_worker.py +0 -0
  261. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/__init__.py +0 -0
  262. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/exceptions.py +0 -0
  263. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/queries.py +0 -0
  264. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/text_processing/topic_modeler.py +0 -0
  265. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/main.py +0 -0
  266. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/__init__.py +0 -0
  267. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/sensitive.py +0 -0
  268. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/version_check.py +0 -0
  269. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/dependency_links.txt +0 -0
  270. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/entry_points.txt +0 -0
  271. {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/top_level.txt +0 -0
@@ -1,9 +1,9 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: qdrant-loader
3
- Version: 1.0.2
3
+ Version: 1.0.4
4
4
  Summary: A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database.
5
5
  Author-email: Martin Papy <martin.papy@cbtw.tech>
6
- License-Expression: GPL-3.0
6
+ License-Expression: Apache-2.0
7
7
  Project-URL: Homepage, https://qdrant-loader.net
8
8
  Project-URL: Documentation, https://qdrant-loader.net/docs/packages/qdrant-loader/README.html
9
9
  Project-URL: Repository, https://github.com/martin-papy/qdrant-loader
@@ -37,7 +37,7 @@ Requires-Dist: pydantic-settings>=2.0.0
37
37
  Requires-Dist: structlog>=23.0.0
38
38
  Requires-Dist: httpx>=0.24.0
39
39
  Requires-Dist: openai>=1.0.0
40
- Requires-Dist: qdrant-loader-core[openai]==1.0.2
40
+ Requires-Dist: qdrant-loader-core[openai]==1.0.4
41
41
  Requires-Dist: qdrant-client>=1.10.0
42
42
  Requires-Dist: PyYAML>=6.0.0
43
43
  Requires-Dist: beautifulsoup4>=4.12.0
@@ -48,7 +48,7 @@ Requires-Dist: langchain-core>=0.3.0
48
48
  Requires-Dist: langchain-community>=0.0.38
49
49
  Requires-Dist: langchain-text-splitters>=0.3.9
50
50
  Requires-Dist: numpy<2.0.0,>=1.26.0
51
- Requires-Dist: GitPython>=3.1.40
51
+ Requires-Dist: GitPython>=3.1.50
52
52
  Requires-Dist: atlassian-python-api>=3.41.0
53
53
  Requires-Dist: tqdm>=4.66.0
54
54
  Requires-Dist: SQLAlchemy>=2.0.0
@@ -63,18 +63,31 @@ Requires-Dist: rank-bm25>=0.2.2
63
63
  Requires-Dist: faiss-cpu>=1.7.4
64
64
  Requires-Dist: psutil>=5.9.0
65
65
  Requires-Dist: tree-sitter-languages>=1.10.0
66
- Requires-Dist: tree-sitter<0.21
66
+ Requires-Dist: tree-sitter<0.27,>=0.25
67
67
  Requires-Dist: markitdown[all]>=0.1.3
68
+ Requires-Dist: docling-slim[standard]>=2.97.0
69
+ Requires-Dist: docling-core>=2.78.0
70
+ Requires-Dist: networkx>=3.0.0
68
71
  Requires-Dist: rich>=13.0.0
69
72
  Requires-Dist: questionary>=2.0.0
70
73
  Requires-Dist: packaging>=21.0
71
74
  Requires-Dist: prometheus-client<1.0.0,>=0.19.0
75
+ Provides-Extra: server
76
+ Requires-Dist: fastapi>=0.104.0; extra == "server"
77
+ Requires-Dist: uvicorn[standard]>=0.24.0; sys_platform != "win32" and extra == "server"
78
+ Requires-Dist: uvicorn>=0.24.0; sys_platform == "win32" and extra == "server"
79
+ Requires-Dist: PyJWT[crypto]>=2.8.0; extra == "server"
80
+ Provides-Extra: postgres
81
+ Requires-Dist: asyncpg>=0.30; extra == "postgres"
82
+ Provides-Extra: dev
83
+ Requires-Dist: testcontainers[postgres]>=4.0; extra == "dev"
84
+ Requires-Dist: asyncpg>=0.30; extra == "dev"
72
85
 
73
86
  # QDrant Loader
74
87
 
75
88
  [![PyPI](https://img.shields.io/pypi/v/qdrant-loader)](https://pypi.org/project/qdrant-loader/)
76
89
  [![Python](https://img.shields.io/pypi/pyversions/qdrant-loader)](https://pypi.org/project/qdrant-loader/)
77
- [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
90
+ [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0)
78
91
 
79
92
  A powerful data ingestion engine that collects and vectorizes technical content from multiple sources for storage in QDrant vector database. Part of the [QDrant Loader monorepo](../../) ecosystem.
80
93
 
@@ -237,7 +250,7 @@ See **[CONTRIBUTING](../../CONTRIBUTING.md)** - Contribution guidelines, develop
237
250
 
238
251
  ## 📄 License
239
252
 
240
- This project is licensed under the GNU GPLv3 - see the [LICENSE](../../LICENSE) file for details.
253
+ This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
241
254
 
242
255
  ---
243
256
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  [![PyPI](https://img.shields.io/pypi/v/qdrant-loader)](https://pypi.org/project/qdrant-loader/)
4
4
  [![Python](https://img.shields.io/pypi/pyversions/qdrant-loader)](https://pypi.org/project/qdrant-loader/)
5
- [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
5
+ [![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0)
6
6
 
7
7
  A powerful data ingestion engine that collects and vectorizes technical content from multiple sources for storage in QDrant vector database. Part of the [QDrant Loader monorepo](../../) ecosystem.
8
8
 
@@ -165,7 +165,7 @@ See **[CONTRIBUTING](../../CONTRIBUTING.md)** - Contribution guidelines, develop
165
165
 
166
166
  ## 📄 License
167
167
 
168
- This project is licensed under the GNU GPLv3 - see the [LICENSE](../../LICENSE) file for details.
168
+ This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
169
169
 
170
170
  ---
171
171
 
@@ -7,11 +7,11 @@ build-backend = "setuptools.build_meta"
7
7
 
8
8
  [project]
9
9
  name = "qdrant-loader"
10
- version = "1.0.2"
10
+ version = "1.0.4"
11
11
  description = "A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database."
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.12"
14
- license = "GPL-3.0"
14
+ license = "Apache-2.0"
15
15
  keywords = [
16
16
  "qdrant",
17
17
  "vector-database",
@@ -35,7 +35,7 @@ dependencies = [
35
35
  "structlog>=23.0.0",
36
36
  "httpx>=0.24.0",
37
37
  "openai>=1.0.0",
38
- "qdrant-loader-core[openai]==1.0.2",
38
+ "qdrant-loader-core[openai]==1.0.4",
39
39
  "qdrant-client>=1.10.0",
40
40
  "PyYAML>=6.0.0",
41
41
  "beautifulsoup4>=4.12.0",
@@ -46,7 +46,7 @@ dependencies = [
46
46
  "langchain-community>=0.0.38",
47
47
  "langchain-text-splitters>=0.3.9",
48
48
  "numpy>=1.26.0,<2.0.0",
49
- "GitPython>=3.1.40",
49
+ "GitPython>=3.1.50",
50
50
  "atlassian-python-api>=3.41.0",
51
51
  "tqdm>=4.66.0",
52
52
  "SQLAlchemy>=2.0.0",
@@ -61,8 +61,11 @@ dependencies = [
61
61
  "faiss-cpu>=1.7.4",
62
62
  "psutil>=5.9.0",
63
63
  "tree-sitter-languages>=1.10.0",
64
- "tree-sitter<0.21",
64
+ "tree-sitter>=0.25,<0.27",
65
65
  "markitdown[all]>=0.1.3",
66
+ "docling-slim[standard]>=2.97.0",
67
+ "docling-core>=2.78.0",
68
+ "networkx>=3.0.0",
66
69
  "rich>=13.0.0",
67
70
  "questionary>=2.0.0",
68
71
  "packaging>=21.0",
@@ -87,6 +90,21 @@ classifiers = [
87
90
  "Typing :: Typed",
88
91
  ]
89
92
 
93
+ [project.optional-dependencies]
94
+ server = [
95
+ "fastapi>=0.104.0",
96
+ "uvicorn[standard]>=0.24.0 ; sys_platform != 'win32'",
97
+ "uvicorn>=0.24.0 ; sys_platform == 'win32'",
98
+ "PyJWT[crypto]>=2.8.0",
99
+ ]
100
+ postgres = [
101
+ "asyncpg>=0.30",
102
+ ]
103
+ dev = [
104
+ "testcontainers[postgres]>=4.0",
105
+ "asyncpg>=0.30",
106
+ ]
107
+
90
108
  [project.urls]
91
109
  Homepage = "https://qdrant-loader.net"
92
110
  Documentation = "https://qdrant-loader.net/docs/packages/qdrant-loader/README.html"
@@ -43,7 +43,14 @@ def __getattr__(name):
43
43
 
44
44
  return QdrantManager
45
45
  else:
46
- raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
46
+ try:
47
+ import importlib
48
+
49
+ return importlib.import_module(f"{__name__}.{name}")
50
+ except ImportError as err:
51
+ raise AttributeError(
52
+ f"module '{__name__}' has no attribute '{name}'"
53
+ ) from err
47
54
 
48
55
 
49
56
  __all__ = [
@@ -0,0 +1,6 @@
1
+ from __future__ import annotations
2
+
3
+ from qdrant_loader.cli.cli import cli
4
+
5
+ if __name__ == "__main__":
6
+ cli()
@@ -11,6 +11,8 @@ from click.utils import echo
11
11
 
12
12
  # async_command is needed at module level for @async_command decorator on commands.
13
13
  from qdrant_loader.cli.asyncio import async_command # noqa: F401
14
+ from qdrant_loader.cli.commands.jobs_cmd import jobs_cmd
15
+ from qdrant_loader.cli.commands.serve_cmd import serve_cmd as serve_command
14
16
  from qdrant_loader.utils.sensitive import sanitize_exception_message
15
17
 
16
18
  # Heavy modules are lazy-imported to keep CLI startup fast.
@@ -77,7 +79,9 @@ def _setup_logging(log_level: str, workspace_config=None) -> None:
77
79
 
78
80
  log_format = "console"
79
81
  log_file = (
80
- str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
82
+ str(workspace_config.logs_path / "cli.log")
83
+ if workspace_config
84
+ else "qdrant-loader.log"
81
85
  )
82
86
  LoggingConfig.setup(level=log_level, format=log_format, file=log_file)
83
87
  # update module-global logger
@@ -96,9 +100,13 @@ def _check_for_updates() -> None:
96
100
  pass
97
101
 
98
102
 
99
- def _setup_workspace(workspace_path: Path):
100
- workspace_config = _setup_workspace_impl(workspace_path)
101
- return workspace_config
103
+ def _setup_workspace(workspace_path):
104
+ try:
105
+ return _setup_workspace_impl(workspace_path)
106
+ except ValueError as e:
107
+ raise ClickException(str(e)) from e
108
+ except Exception as e: # pragma: no cover - handled by CLI tests
109
+ raise ClickException(f"Failed to setup workspace: {str(e)!s}") from e
102
110
 
103
111
 
104
112
  @group(name="qdrant-loader")
@@ -120,6 +128,10 @@ def cli(log_level: str = "INFO") -> None:
120
128
  _check_for_updates()
121
129
 
122
130
 
131
+ cli.add_command(serve_command)
132
+ cli.add_command(jobs_cmd)
133
+
134
+
123
135
  def _create_database_directory(path: Path) -> bool:
124
136
  """Create database directory with user confirmation.
125
137
 
@@ -350,7 +362,12 @@ async def _cancel_all_tasks():
350
362
  @option(
351
363
  "--force",
352
364
  is_flag=True,
353
- help="Force processing of all documents, bypassing change detection. Warning: May significantly increase processing time and costs.",
365
+ help="Force processing of all documents, bypassing change detection and ignoring checkpoints. Warning: May significantly increase processing time and costs.",
366
+ )
367
+ @option(
368
+ "--resume/--no-resume",
369
+ default=True,
370
+ help="Resume from the last checkpoint if available (default: enabled). Use --no-resume to start fresh from the beginning.",
354
371
  )
355
372
  @async_command
356
373
  async def ingest(
@@ -363,13 +380,20 @@ async def ingest(
363
380
  log_level: str,
364
381
  profile: bool,
365
382
  force: bool,
383
+ resume: bool,
366
384
  ):
367
385
  """Ingest documents from configured sources.
368
386
 
369
387
  Examples:
370
- # Ingest all projects
388
+ # Ingest all projects (with checkpoint resume if available)
371
389
  qdrant-loader ingest
372
390
 
391
+ # Ingest with checkpoint resume disabled
392
+ qdrant-loader ingest --no-resume
393
+
394
+ # Force full re-ingestion (ignores checkpoints)
395
+ qdrant-loader ingest --force
396
+
373
397
  # Ingest specific project
374
398
  qdrant-loader ingest --project my-project
375
399
 
@@ -397,9 +421,27 @@ async def ingest(
397
421
  log_level,
398
422
  profile,
399
423
  force,
424
+ resume,
400
425
  )
401
426
 
402
427
 
428
+ async def _start_webhook_server(
429
+ workspace: Path | None,
430
+ config: Path | None,
431
+ env: Path | None,
432
+ host: str,
433
+ port: int,
434
+ log_level: str,
435
+ ) -> None:
436
+ """Start webhook server for receiving connector events and triggering ingestion.
437
+
438
+ Internal function called by the future `serve` command. Not exposed as a user CLI command in v1.1.
439
+ """
440
+ from qdrant_loader.cli.commands.webhook_cmd import run_webhook_command
441
+
442
+ await run_webhook_command(workspace, config, env, host, port, log_level)
443
+
444
+
403
445
  @cli.command()
404
446
  @option(
405
447
  "--output-dir",
@@ -10,10 +10,25 @@ def __getattr__(name: str):
10
10
  from .ingest import run_pipeline_ingestion
11
11
 
12
12
  return run_pipeline_ingestion
13
+ if name == "serve_cmd":
14
+ from .serve_cmd import serve_cmd
15
+
16
+ return serve_cmd
17
+ if name == "jobs_cmd":
18
+ from .jobs_cmd import jobs_cmd
19
+
20
+ return jobs_cmd
21
+ if name == "run_webhook_command":
22
+ from .webhook_cmd import run_webhook_command
23
+
24
+ return run_webhook_command
13
25
  raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
14
26
 
15
27
 
16
28
  __all__ = [
17
29
  "run_init",
18
30
  "run_pipeline_ingestion",
31
+ "serve_cmd",
32
+ "jobs_cmd",
33
+ "run_webhook_command",
19
34
  ]
@@ -40,7 +40,9 @@ def run_show_config(
40
40
 
41
41
  # Setup/reconfigure logging once with workspace support
42
42
  log_file = (
43
- str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
43
+ str(workspace_config.logs_path / "config.log")
44
+ if workspace_config
45
+ else "qdrant-loader.log"
44
46
  )
45
47
  if getattr(LoggingConfig, "reconfigure", None): # Core supports reconfigure
46
48
  if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined]
@@ -18,7 +18,9 @@ def run_config_command(
18
18
  # Maintain test expectation: log via workspace-logger as before
19
19
  workspace_config = _setup_workspace_impl(workspace) if workspace else None
20
20
  log_file = (
21
- str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
21
+ str(workspace_config.logs_path / "config.log")
22
+ if workspace_config
23
+ else "qdrant-loader.log"
22
24
  )
23
25
  if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined]
24
26
  if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined]
@@ -15,6 +15,7 @@ async def run_pipeline_ingestion(
15
15
  source_type: str | None,
16
16
  source: str | None,
17
17
  force: bool,
18
+ resume: bool = True,
18
19
  metrics_dir: str | None = None,
19
20
  ) -> None:
20
21
  from qdrant_loader.core.async_ingestion_pipeline import AsyncIngestionPipeline
@@ -32,6 +33,7 @@ async def run_pipeline_ingestion(
32
33
  source_type=source_type,
33
34
  source=source,
34
35
  force=force,
36
+ resume=resume,
35
37
  )
36
38
  except Exception as e:
37
39
  ingestion_error = e
@@ -36,8 +36,9 @@ async def run_ingest_command(
36
36
  log_level: str,
37
37
  profile: bool,
38
38
  force: bool,
39
+ resume: bool = True,
39
40
  ) -> None:
40
- """Implementation for the `ingest` CLI command with identical behavior."""
41
+ """Implementation for the `ingest` CLI command with checkpoint resume support (WS-2)."""
41
42
 
42
43
  ingest_start_time = time.perf_counter()
43
44
 
@@ -52,7 +53,9 @@ async def run_ingest_command(
52
53
 
53
54
  # Setup/reconfigure logging with workspace support
54
55
  log_file = (
55
- str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
56
+ str(workspace_config.logs_path / "ingest.log")
57
+ if workspace_config
58
+ else "qdrant-loader.log"
56
59
  )
57
60
  if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined]
58
61
  if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined]
@@ -87,6 +90,7 @@ async def run_ingest_command(
87
90
  source_type=source_type,
88
91
  source=source,
89
92
  force=force,
93
+ resume=resume,
90
94
  metrics_dir=(
91
95
  str(workspace_config.metrics_path) if workspace_config else None
92
96
  ),
@@ -83,7 +83,7 @@ async def run_init_command(
83
83
 
84
84
  # Setup logging again with workspace-aware file path
85
85
  log_file = (
86
- str(workspace_config.logs_path)
86
+ str(workspace_config.logs_path / "init.log")
87
87
  if workspace_config
88
88
  else "qdrant-loader.log"
89
89
  )
@@ -0,0 +1,259 @@
1
+ """
2
+ qdrant-loader jobs admin CLI commands.
3
+
4
+ Subcommands: list [--status], retry <id>, trigger --source-type --source --mode, cancel <id>.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ import json
11
+ from pathlib import Path
12
+
13
+ import click
14
+ from click.exceptions import ClickException
15
+ from click.types import Choice
16
+ from click.types import Path as ClickPath
17
+
18
+ from qdrant_loader.core.worker.job_types import JobType
19
+
20
+
21
+ def _init_queue(workspace: Path | None, config: Path | None, env: Path | None):
22
+ """Load config, build StateManager. Returns state_manager."""
23
+ from qdrant_loader.cli.config_loader import (
24
+ load_config_with_workspace,
25
+ setup_workspace,
26
+ )
27
+ from qdrant_loader.config import get_global_config
28
+ from qdrant_loader.config.workspace import validate_workspace_flags
29
+ from qdrant_loader.core.state.state_manager import StateManager
30
+
31
+ # Validate mutually exclusive flags early
32
+ try:
33
+ validate_workspace_flags(workspace, config, env)
34
+ except ValueError as e:
35
+ raise ClickException(str(e))
36
+
37
+ if workspace:
38
+ ws_config = setup_workspace(workspace)
39
+ load_config_with_workspace(workspace_config=ws_config, skip_validation=True)
40
+ else:
41
+ resolved_config = config
42
+ if resolved_config is None:
43
+ default_config = Path("config.yaml")
44
+ resolved_config = default_config if default_config.exists() else None
45
+ if resolved_config is None:
46
+ raise ClickException("No config found. Use --workspace or --config.")
47
+ load_config_with_workspace(
48
+ workspace_config=None,
49
+ config_path=resolved_config,
50
+ env_path=env,
51
+ skip_validation=True,
52
+ )
53
+
54
+ global_config = get_global_config()
55
+ state_manager = StateManager(global_config.state_management)
56
+ return state_manager
57
+
58
+
59
+ async def _run_with_queue(workspace, config, env, coro_factory):
60
+ """Initialize queue, run coro_factory(queue), dispose state manager."""
61
+ state_manager = _init_queue(workspace, config, env)
62
+ await state_manager.initialize()
63
+ queue_instance = None
64
+ try:
65
+ from qdrant_loader.core.worker.queue import SQLiteJobQueue
66
+
67
+ queue_instance = SQLiteJobQueue(
68
+ state_manager.session_factory,
69
+ db_op_lock=state_manager.queue_db_op_lock,
70
+ )
71
+ return await coro_factory(queue_instance)
72
+ finally:
73
+ await state_manager.dispose()
74
+
75
+
76
+ # ────────────────────────────────────────────────
77
+ # Click group
78
+ # ────────────────────────────────────────────────
79
+
80
+
81
+ @click.group("jobs")
82
+ def jobs_cmd():
83
+ """Inspect and manage background ingestion jobs."""
84
+
85
+
86
+ def _common_options(fn):
87
+ """Decorator that adds --workspace, --config, --env options."""
88
+ fn = click.option(
89
+ "--workspace",
90
+ type=ClickPath(path_type=Path),
91
+ default=None,
92
+ help="Workspace directory.",
93
+ )(fn)
94
+ fn = click.option(
95
+ "--config",
96
+ "config_path",
97
+ type=ClickPath(path_type=Path),
98
+ default=None,
99
+ help="Path to config.yaml.",
100
+ )(fn)
101
+ fn = click.option(
102
+ "--env",
103
+ "env_path",
104
+ type=ClickPath(path_type=Path),
105
+ default=None,
106
+ help="Path to .env file.",
107
+ )(fn)
108
+ return fn
109
+
110
+
111
+ # ────────────────────────────────────────────────
112
+ # jobs list
113
+ # ────────────────────────────────────────────────
114
+
115
+
116
+ @jobs_cmd.command("list")
117
+ @click.option(
118
+ "--status",
119
+ type=Choice(
120
+ ["pending", "running", "done", "failed", "cancelled"], case_sensitive=False
121
+ ),
122
+ default=None,
123
+ help="Filter by job status.",
124
+ )
125
+ @click.option(
126
+ "--limit",
127
+ type=click.IntRange(min=1, max=1000),
128
+ default=50,
129
+ show_default=True,
130
+ help="Max rows to return.",
131
+ )
132
+ @click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
133
+ @_common_options
134
+ def jobs_list(status, limit, output_json, workspace, config_path, env_path):
135
+ """List jobs, optionally filtered by status."""
136
+
137
+ async def _list(queue):
138
+ jobs = await queue.list(status=status, limit=limit)
139
+ if output_json:
140
+ rows = []
141
+ for j in jobs:
142
+ rows.append(
143
+ {
144
+ "id": j.id,
145
+ "type": j.type,
146
+ "status": j.status,
147
+ "attempts": j.attempts,
148
+ "enqueued_at": (
149
+ j.enqueued_at.isoformat() if j.enqueued_at else None
150
+ ),
151
+ "started_at": (
152
+ j.started_at.isoformat() if j.started_at else None
153
+ ),
154
+ "finished_at": (
155
+ j.finished_at.isoformat() if j.finished_at else None
156
+ ),
157
+ "last_error": j.last_error,
158
+ "payload": json.loads(j.payload_json) if j.payload_json else {},
159
+ }
160
+ )
161
+ click.echo(json.dumps(rows, indent=2))
162
+ else:
163
+ if not jobs:
164
+ click.echo("No jobs found.")
165
+ return
166
+ header = f"{'ID':>6} {'STATUS':<10} {'TYPE':<20} {'ATTEMPTS':>8} {'ENQUEUED_AT':<27} LAST_ERROR"
167
+ click.echo(header)
168
+ click.echo("-" * len(header))
169
+ for j in jobs:
170
+ enq = j.enqueued_at.isoformat() if j.enqueued_at else ""
171
+ err = (j.last_error or "")[:40]
172
+ click.echo(
173
+ f"{j.id:>6} {j.status:<10} {j.type:<20} {j.attempts:>8} {enq:<27} {err}"
174
+ )
175
+
176
+ asyncio.run(_run_with_queue(workspace, config_path, env_path, _list))
177
+
178
+
179
+ # ────────────────────────────────────────────────
180
+ # jobs retry
181
+ # ────────────────────────────────────────────────
182
+
183
+
184
+ @jobs_cmd.command("retry")
185
+ @click.argument("job_id", type=int)
186
+ @_common_options
187
+ def jobs_retry(job_id, workspace, config_path, env_path):
188
+ """Reset a failed or done job back to pending."""
189
+
190
+ async def _retry(queue):
191
+ ok = await queue.reset_to_pending(job_id)
192
+ if ok:
193
+ click.echo(f"Job {job_id} reset to pending.")
194
+ else:
195
+ raise ClickException(
196
+ f"Job {job_id} not found or is not in failed/done state."
197
+ )
198
+
199
+ asyncio.run(_run_with_queue(workspace, config_path, env_path, _retry))
200
+
201
+
202
+ # ────────────────────────────────────────────────
203
+ # jobs trigger
204
+ # ────────────────────────────────────────────────
205
+
206
+
207
+ @jobs_cmd.command("trigger")
208
+ @click.option(
209
+ "--source-type", required=True, help="Source type (e.g. git, confluence)."
210
+ )
211
+ @click.option("--source", required=True, help="Source name as configured.")
212
+ @click.option(
213
+ "--mode",
214
+ type=Choice(["bulk", "incremental"], case_sensitive=False),
215
+ required=True,
216
+ help="Ingestion mode.",
217
+ )
218
+ @click.option("--project", "project_id", required=True, help="Project ID.")
219
+ @_common_options
220
+ def jobs_trigger(
221
+ source_type, source, mode, project_id, workspace, config_path, env_path
222
+ ):
223
+ """Enqueue a new ingestion job immediately."""
224
+
225
+ async def _trigger(queue):
226
+ job_type = (
227
+ JobType.BULK_INGEST if mode.lower() == "bulk" else JobType.INCREMENTAL_PULL
228
+ )
229
+ payload = {
230
+ "project_id": project_id,
231
+ "source_type": source_type,
232
+ "source": source,
233
+ "source_lock": f"{project_id}:{source_type}:{source}",
234
+ }
235
+ job = await queue.enqueue(job_type, payload)
236
+ click.echo(f"Enqueued job {job.id} (type={job_type}, status={job.status}).")
237
+
238
+ asyncio.run(_run_with_queue(workspace, config_path, env_path, _trigger))
239
+
240
+
241
+ # ────────────────────────────────────────────────
242
+ # jobs cancel
243
+ # ────────────────────────────────────────────────
244
+
245
+
246
+ @jobs_cmd.command("cancel")
247
+ @click.argument("job_id", type=int)
248
+ @_common_options
249
+ def jobs_cancel(job_id, workspace, config_path, env_path):
250
+ """Cancel a pending job."""
251
+
252
+ async def _cancel(queue):
253
+ ok = await queue.cancel(job_id)
254
+ if ok:
255
+ click.echo(f"Job {job_id} cancelled.")
256
+ else:
257
+ raise ClickException(f"Job {job_id} not found or is not in pending state.")
258
+
259
+ asyncio.run(_run_with_queue(workspace, config_path, env_path, _cancel))