qdrant-loader 1.0.2__tar.gz → 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/PKG-INFO +20 -7
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/README.md +2 -2
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/pyproject.toml +23 -5
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/__init__.py +8 -1
- qdrant_loader-1.0.4/src/qdrant_loader/cli/__main__.py +6 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/cli.py +48 -6
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/__init__.py +15 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/config.py +3 -1
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/config_cmd.py +3 -1
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/ingest.py +2 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/ingest_cmd.py +6 -2
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/init_cmd.py +1 -1
- qdrant_loader-1.0.4/src/qdrant_loader/cli/commands/jobs_cmd.py +259 -0
- qdrant_loader-1.0.4/src/qdrant_loader/cli/commands/serve_cmd.py +339 -0
- qdrant_loader-1.0.4/src/qdrant_loader/cli/commands/webhook_cmd.py +76 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/logging_utils.py +1 -1
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/__init__.py +45 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/chunking.py +33 -0
- qdrant_loader-1.0.4/src/qdrant_loader/config/concurrency.py +36 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/embedding.py +10 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/global_config.py +21 -0
- qdrant_loader-1.0.4/src/qdrant_loader/config/graph.py +63 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/state.py +19 -0
- qdrant_loader-1.0.4/src/qdrant_loader/config/workers.py +202 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/workspace.py +1 -1
- qdrant_loader-1.0.4/src/qdrant_loader/connectors/base.py +113 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/connector.py +237 -137
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/pagination.py +20 -4
- qdrant_loader-1.0.4/src/qdrant_loader/connectors/factory.py +27 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/connector.py +47 -11
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/cloud_connector.py +30 -9
- qdrant_loader-1.0.4/src/qdrant_loader/connectors/jira/config.py +317 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/connector.py +160 -69
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/data_center_connector.py +32 -9
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/mappers.py +94 -13
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/models.py +27 -1
- qdrant_loader-1.0.4/src/qdrant_loader/connectors/localfile/connector.py +263 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/connector.py +69 -32
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/async_ingestion_pipeline.py +61 -34
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/chunking_service.py +12 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/__init__.py +48 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/chunker.py +72 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/config.py +127 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/docling_chunker.py +155 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/exceptions.py +26 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/mapper.py +111 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/outcome.py +27 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/structure.py +117 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/docling/tokenizer.py +81 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/__init__.py +4 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base_strategy.py +8 -1
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/strategy/docling_strategy.py +156 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/chunk_processor.py +10 -61
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/section_splitter.py +14 -8
- qdrant_loader-1.0.4/src/qdrant_loader/core/chunking/strategy/markdown/splitters/row_kv_excel.py +213 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/__init__.py +39 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/config.py +126 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/docling_engine.py +67 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/engine.py +56 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/formats.py +54 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/options.py +141 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/outcome.py +100 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/postprocess.py +131 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/service.py +257 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/conversion/settings.py +101 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/document.py +32 -3
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/embedding/embedding_service.py +18 -2
- qdrant_loader-1.0.4/src/qdrant_loader/core/file_conversion/clean_xlsx_converter.py +164 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/conversion_config.py +17 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/file_converter.py +15 -6
- qdrant_loader-1.0.4/src/qdrant_loader/core/file_conversion/sub_table_detector.py +110 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/file_conversion/xlsx_markdown_format.py +23 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/document_pipeline.py +356 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/factory.py +1 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/orchestrator.py +867 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/source_processor.py +87 -4
- qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/workers/embedding_worker.py +224 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/pipeline/workers/upsert_worker.py +443 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/project_manager.py +33 -4
- qdrant_loader-1.0.4/src/qdrant_loader/core/qdrant_manager.py +568 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/state/checkpoint_manager.py +286 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/document_state_manager.py +1 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/models.py +53 -4
- qdrant_loader-1.0.4/src/qdrant_loader/core/state/session.py +73 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/state_change_detector.py +80 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/state_manager.py +176 -1
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/transitions.py +203 -100
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/utils.py +32 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/text_processing/chunk_enricher.py +91 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/text_processing/semantic_analyzer.py +170 -88
- qdrant_loader-1.0.4/src/qdrant_loader/core/text_processing/spacy_model_cache.py +51 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/text_processing/text_processor.py +30 -24
- qdrant_loader-1.0.4/src/qdrant_loader/core/worker/__init__.py +30 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/worker/handlers.py +215 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/worker/job_types.py +29 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/worker/pool.py +381 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/worker/queue.py +442 -0
- qdrant_loader-1.0.4/src/qdrant_loader/core/worker/scheduler.py +183 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/logging.py +74 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/__init__.py +5 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/auth.py +273 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/event_processor.py +216 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/handlers.py +194 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/queue_backend.py +174 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/server.py +327 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/single_event_handler.py +67 -0
- qdrant_loader-1.0.4/src/qdrant_loader/webhooks/worker.py +86 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/PKG-INFO +20 -7
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/SOURCES.txt +49 -1
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/requires.txt +23 -3
- qdrant_loader-1.0.2/src/qdrant_loader/connectors/base.py +0 -48
- qdrant_loader-1.0.2/src/qdrant_loader/connectors/factory.py +0 -15
- qdrant_loader-1.0.2/src/qdrant_loader/connectors/jira/config.py +0 -151
- qdrant_loader-1.0.2/src/qdrant_loader/connectors/localfile/connector.py +0 -206
- qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/document_pipeline.py +0 -94
- qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/orchestrator.py +0 -400
- qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/workers/embedding_worker.py +0 -158
- qdrant_loader-1.0.2/src/qdrant_loader/core/pipeline/workers/upsert_worker.py +0 -287
- qdrant_loader-1.0.2/src/qdrant_loader/core/qdrant_manager.py +0 -384
- qdrant_loader-1.0.2/src/qdrant_loader/core/state/session.py +0 -37
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/setup.cfg +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/async_utils.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/asyncio.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/init.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/list_cmd.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/status_cmd.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/project/validate_cmd.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/commands/setup_cmd.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/config_loader.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/path_utils.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/project_commands.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/update_check.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/cli/version.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/base.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/error_formatter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/models.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/qdrant.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/source_config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/sources.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/types.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config/validator.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/attachments.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/auth.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/confluence/mappers.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/exceptions.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/adapter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/file_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/git/operations.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/http.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/jira/auth.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/file_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/localfile/metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/crawler.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/http.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/publicdocs/parsers.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/registry.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/attachments/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/attachments/metadata.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/attachments/reader.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/client.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/errors.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/policy.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/connectors/shared/http/rate_limit.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/attachment_downloader.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/progress_tracker.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/chunk_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/document_parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/base/section_splitter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_chunk_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_document_parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/code_section_splitter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/complexity.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/dependencies.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/documentation.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/entities.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/language_specific.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/maintainability.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/patterns.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/performance.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/security.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/metadata/testing.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/common.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/python_ast.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/parser/tree_sitter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/analysis.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/quality.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code/processor/utils.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/code_strategy.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_chunk_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_document_parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default/text_section_splitter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/default_strategy.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_chunk_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_document_parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html/html_section_splitter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/html_strategy.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_chunk_processor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_document_parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json/json_section_splitter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/json_strategy.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/document_parser.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/markdown_strategy.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/metadata_extractor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/base.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/excel.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/fallback.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/chunking/strategy/markdown/splitters/standard.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/embedding/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/embedding_service.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/exceptions.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/file_conversion/file_detector.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/init_collection.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/batch_summary.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/ingestion_metrics.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/processing_stats.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/prometheus_metrics.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/monitoring/resource_monitor.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/config.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/resource_manager.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/source_filter.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/workers/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/workers/base_worker.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/pipeline/workers/chunking_worker.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/exceptions.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/state/queries.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/core/text_processing/topic_modeler.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/main.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/__init__.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/sensitive.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader/utils/version_check.py +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/dependency_links.txt +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/entry_points.txt +0 -0
- {qdrant_loader-1.0.2 → qdrant_loader-1.0.4}/src/qdrant_loader.egg-info/top_level.txt +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: qdrant-loader
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.4
|
|
4
4
|
Summary: A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database.
|
|
5
5
|
Author-email: Martin Papy <martin.papy@cbtw.tech>
|
|
6
|
-
License-Expression:
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
7
|
Project-URL: Homepage, https://qdrant-loader.net
|
|
8
8
|
Project-URL: Documentation, https://qdrant-loader.net/docs/packages/qdrant-loader/README.html
|
|
9
9
|
Project-URL: Repository, https://github.com/martin-papy/qdrant-loader
|
|
@@ -37,7 +37,7 @@ Requires-Dist: pydantic-settings>=2.0.0
|
|
|
37
37
|
Requires-Dist: structlog>=23.0.0
|
|
38
38
|
Requires-Dist: httpx>=0.24.0
|
|
39
39
|
Requires-Dist: openai>=1.0.0
|
|
40
|
-
Requires-Dist: qdrant-loader-core[openai]==1.0.
|
|
40
|
+
Requires-Dist: qdrant-loader-core[openai]==1.0.4
|
|
41
41
|
Requires-Dist: qdrant-client>=1.10.0
|
|
42
42
|
Requires-Dist: PyYAML>=6.0.0
|
|
43
43
|
Requires-Dist: beautifulsoup4>=4.12.0
|
|
@@ -48,7 +48,7 @@ Requires-Dist: langchain-core>=0.3.0
|
|
|
48
48
|
Requires-Dist: langchain-community>=0.0.38
|
|
49
49
|
Requires-Dist: langchain-text-splitters>=0.3.9
|
|
50
50
|
Requires-Dist: numpy<2.0.0,>=1.26.0
|
|
51
|
-
Requires-Dist: GitPython>=3.1.
|
|
51
|
+
Requires-Dist: GitPython>=3.1.50
|
|
52
52
|
Requires-Dist: atlassian-python-api>=3.41.0
|
|
53
53
|
Requires-Dist: tqdm>=4.66.0
|
|
54
54
|
Requires-Dist: SQLAlchemy>=2.0.0
|
|
@@ -63,18 +63,31 @@ Requires-Dist: rank-bm25>=0.2.2
|
|
|
63
63
|
Requires-Dist: faiss-cpu>=1.7.4
|
|
64
64
|
Requires-Dist: psutil>=5.9.0
|
|
65
65
|
Requires-Dist: tree-sitter-languages>=1.10.0
|
|
66
|
-
Requires-Dist: tree-sitter<0.
|
|
66
|
+
Requires-Dist: tree-sitter<0.27,>=0.25
|
|
67
67
|
Requires-Dist: markitdown[all]>=0.1.3
|
|
68
|
+
Requires-Dist: docling-slim[standard]>=2.97.0
|
|
69
|
+
Requires-Dist: docling-core>=2.78.0
|
|
70
|
+
Requires-Dist: networkx>=3.0.0
|
|
68
71
|
Requires-Dist: rich>=13.0.0
|
|
69
72
|
Requires-Dist: questionary>=2.0.0
|
|
70
73
|
Requires-Dist: packaging>=21.0
|
|
71
74
|
Requires-Dist: prometheus-client<1.0.0,>=0.19.0
|
|
75
|
+
Provides-Extra: server
|
|
76
|
+
Requires-Dist: fastapi>=0.104.0; extra == "server"
|
|
77
|
+
Requires-Dist: uvicorn[standard]>=0.24.0; sys_platform != "win32" and extra == "server"
|
|
78
|
+
Requires-Dist: uvicorn>=0.24.0; sys_platform == "win32" and extra == "server"
|
|
79
|
+
Requires-Dist: PyJWT[crypto]>=2.8.0; extra == "server"
|
|
80
|
+
Provides-Extra: postgres
|
|
81
|
+
Requires-Dist: asyncpg>=0.30; extra == "postgres"
|
|
82
|
+
Provides-Extra: dev
|
|
83
|
+
Requires-Dist: testcontainers[postgres]>=4.0; extra == "dev"
|
|
84
|
+
Requires-Dist: asyncpg>=0.30; extra == "dev"
|
|
72
85
|
|
|
73
86
|
# QDrant Loader
|
|
74
87
|
|
|
75
88
|
[](https://pypi.org/project/qdrant-loader/)
|
|
76
89
|
[](https://pypi.org/project/qdrant-loader/)
|
|
77
|
-
[](https://www.apache.org/licenses/LICENSE-2.0)
|
|
78
91
|
|
|
79
92
|
A powerful data ingestion engine that collects and vectorizes technical content from multiple sources for storage in QDrant vector database. Part of the [QDrant Loader monorepo](../../) ecosystem.
|
|
80
93
|
|
|
@@ -237,7 +250,7 @@ See **[CONTRIBUTING](../../CONTRIBUTING.md)** - Contribution guidelines, develop
|
|
|
237
250
|
|
|
238
251
|
## 📄 License
|
|
239
252
|
|
|
240
|
-
This project is licensed under the
|
|
253
|
+
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
|
241
254
|
|
|
242
255
|
---
|
|
243
256
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/qdrant-loader/)
|
|
4
4
|
[](https://pypi.org/project/qdrant-loader/)
|
|
5
|
-
[](https://www.apache.org/licenses/LICENSE-2.0)
|
|
6
6
|
|
|
7
7
|
A powerful data ingestion engine that collects and vectorizes technical content from multiple sources for storage in QDrant vector database. Part of the [QDrant Loader monorepo](../../) ecosystem.
|
|
8
8
|
|
|
@@ -165,7 +165,7 @@ See **[CONTRIBUTING](../../CONTRIBUTING.md)** - Contribution guidelines, develop
|
|
|
165
165
|
|
|
166
166
|
## 📄 License
|
|
167
167
|
|
|
168
|
-
This project is licensed under the
|
|
168
|
+
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
|
169
169
|
|
|
170
170
|
---
|
|
171
171
|
|
|
@@ -7,11 +7,11 @@ build-backend = "setuptools.build_meta"
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "qdrant-loader"
|
|
10
|
-
version = "1.0.
|
|
10
|
+
version = "1.0.4"
|
|
11
11
|
description = "A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database."
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
requires-python = ">=3.12"
|
|
14
|
-
license = "
|
|
14
|
+
license = "Apache-2.0"
|
|
15
15
|
keywords = [
|
|
16
16
|
"qdrant",
|
|
17
17
|
"vector-database",
|
|
@@ -35,7 +35,7 @@ dependencies = [
|
|
|
35
35
|
"structlog>=23.0.0",
|
|
36
36
|
"httpx>=0.24.0",
|
|
37
37
|
"openai>=1.0.0",
|
|
38
|
-
"qdrant-loader-core[openai]==1.0.
|
|
38
|
+
"qdrant-loader-core[openai]==1.0.4",
|
|
39
39
|
"qdrant-client>=1.10.0",
|
|
40
40
|
"PyYAML>=6.0.0",
|
|
41
41
|
"beautifulsoup4>=4.12.0",
|
|
@@ -46,7 +46,7 @@ dependencies = [
|
|
|
46
46
|
"langchain-community>=0.0.38",
|
|
47
47
|
"langchain-text-splitters>=0.3.9",
|
|
48
48
|
"numpy>=1.26.0,<2.0.0",
|
|
49
|
-
"GitPython>=3.1.
|
|
49
|
+
"GitPython>=3.1.50",
|
|
50
50
|
"atlassian-python-api>=3.41.0",
|
|
51
51
|
"tqdm>=4.66.0",
|
|
52
52
|
"SQLAlchemy>=2.0.0",
|
|
@@ -61,8 +61,11 @@ dependencies = [
|
|
|
61
61
|
"faiss-cpu>=1.7.4",
|
|
62
62
|
"psutil>=5.9.0",
|
|
63
63
|
"tree-sitter-languages>=1.10.0",
|
|
64
|
-
"tree-sitter
|
|
64
|
+
"tree-sitter>=0.25,<0.27",
|
|
65
65
|
"markitdown[all]>=0.1.3",
|
|
66
|
+
"docling-slim[standard]>=2.97.0",
|
|
67
|
+
"docling-core>=2.78.0",
|
|
68
|
+
"networkx>=3.0.0",
|
|
66
69
|
"rich>=13.0.0",
|
|
67
70
|
"questionary>=2.0.0",
|
|
68
71
|
"packaging>=21.0",
|
|
@@ -87,6 +90,21 @@ classifiers = [
|
|
|
87
90
|
"Typing :: Typed",
|
|
88
91
|
]
|
|
89
92
|
|
|
93
|
+
[project.optional-dependencies]
|
|
94
|
+
server = [
|
|
95
|
+
"fastapi>=0.104.0",
|
|
96
|
+
"uvicorn[standard]>=0.24.0 ; sys_platform != 'win32'",
|
|
97
|
+
"uvicorn>=0.24.0 ; sys_platform == 'win32'",
|
|
98
|
+
"PyJWT[crypto]>=2.8.0",
|
|
99
|
+
]
|
|
100
|
+
postgres = [
|
|
101
|
+
"asyncpg>=0.30",
|
|
102
|
+
]
|
|
103
|
+
dev = [
|
|
104
|
+
"testcontainers[postgres]>=4.0",
|
|
105
|
+
"asyncpg>=0.30",
|
|
106
|
+
]
|
|
107
|
+
|
|
90
108
|
[project.urls]
|
|
91
109
|
Homepage = "https://qdrant-loader.net"
|
|
92
110
|
Documentation = "https://qdrant-loader.net/docs/packages/qdrant-loader/README.html"
|
|
@@ -43,7 +43,14 @@ def __getattr__(name):
|
|
|
43
43
|
|
|
44
44
|
return QdrantManager
|
|
45
45
|
else:
|
|
46
|
-
|
|
46
|
+
try:
|
|
47
|
+
import importlib
|
|
48
|
+
|
|
49
|
+
return importlib.import_module(f"{__name__}.{name}")
|
|
50
|
+
except ImportError as err:
|
|
51
|
+
raise AttributeError(
|
|
52
|
+
f"module '{__name__}' has no attribute '{name}'"
|
|
53
|
+
) from err
|
|
47
54
|
|
|
48
55
|
|
|
49
56
|
__all__ = [
|
|
@@ -11,6 +11,8 @@ from click.utils import echo
|
|
|
11
11
|
|
|
12
12
|
# async_command is needed at module level for @async_command decorator on commands.
|
|
13
13
|
from qdrant_loader.cli.asyncio import async_command # noqa: F401
|
|
14
|
+
from qdrant_loader.cli.commands.jobs_cmd import jobs_cmd
|
|
15
|
+
from qdrant_loader.cli.commands.serve_cmd import serve_cmd as serve_command
|
|
14
16
|
from qdrant_loader.utils.sensitive import sanitize_exception_message
|
|
15
17
|
|
|
16
18
|
# Heavy modules are lazy-imported to keep CLI startup fast.
|
|
@@ -77,7 +79,9 @@ def _setup_logging(log_level: str, workspace_config=None) -> None:
|
|
|
77
79
|
|
|
78
80
|
log_format = "console"
|
|
79
81
|
log_file = (
|
|
80
|
-
str(workspace_config.logs_path
|
|
82
|
+
str(workspace_config.logs_path / "cli.log")
|
|
83
|
+
if workspace_config
|
|
84
|
+
else "qdrant-loader.log"
|
|
81
85
|
)
|
|
82
86
|
LoggingConfig.setup(level=log_level, format=log_format, file=log_file)
|
|
83
87
|
# update module-global logger
|
|
@@ -96,9 +100,13 @@ def _check_for_updates() -> None:
|
|
|
96
100
|
pass
|
|
97
101
|
|
|
98
102
|
|
|
99
|
-
def _setup_workspace(workspace_path
|
|
100
|
-
|
|
101
|
-
|
|
103
|
+
def _setup_workspace(workspace_path):
|
|
104
|
+
try:
|
|
105
|
+
return _setup_workspace_impl(workspace_path)
|
|
106
|
+
except ValueError as e:
|
|
107
|
+
raise ClickException(str(e)) from e
|
|
108
|
+
except Exception as e: # pragma: no cover - handled by CLI tests
|
|
109
|
+
raise ClickException(f"Failed to setup workspace: {str(e)!s}") from e
|
|
102
110
|
|
|
103
111
|
|
|
104
112
|
@group(name="qdrant-loader")
|
|
@@ -120,6 +128,10 @@ def cli(log_level: str = "INFO") -> None:
|
|
|
120
128
|
_check_for_updates()
|
|
121
129
|
|
|
122
130
|
|
|
131
|
+
cli.add_command(serve_command)
|
|
132
|
+
cli.add_command(jobs_cmd)
|
|
133
|
+
|
|
134
|
+
|
|
123
135
|
def _create_database_directory(path: Path) -> bool:
|
|
124
136
|
"""Create database directory with user confirmation.
|
|
125
137
|
|
|
@@ -350,7 +362,12 @@ async def _cancel_all_tasks():
|
|
|
350
362
|
@option(
|
|
351
363
|
"--force",
|
|
352
364
|
is_flag=True,
|
|
353
|
-
help="Force processing of all documents, bypassing change detection. Warning: May significantly increase processing time and costs.",
|
|
365
|
+
help="Force processing of all documents, bypassing change detection and ignoring checkpoints. Warning: May significantly increase processing time and costs.",
|
|
366
|
+
)
|
|
367
|
+
@option(
|
|
368
|
+
"--resume/--no-resume",
|
|
369
|
+
default=True,
|
|
370
|
+
help="Resume from the last checkpoint if available (default: enabled). Use --no-resume to start fresh from the beginning.",
|
|
354
371
|
)
|
|
355
372
|
@async_command
|
|
356
373
|
async def ingest(
|
|
@@ -363,13 +380,20 @@ async def ingest(
|
|
|
363
380
|
log_level: str,
|
|
364
381
|
profile: bool,
|
|
365
382
|
force: bool,
|
|
383
|
+
resume: bool,
|
|
366
384
|
):
|
|
367
385
|
"""Ingest documents from configured sources.
|
|
368
386
|
|
|
369
387
|
Examples:
|
|
370
|
-
# Ingest all projects
|
|
388
|
+
# Ingest all projects (with checkpoint resume if available)
|
|
371
389
|
qdrant-loader ingest
|
|
372
390
|
|
|
391
|
+
# Ingest with checkpoint resume disabled
|
|
392
|
+
qdrant-loader ingest --no-resume
|
|
393
|
+
|
|
394
|
+
# Force full re-ingestion (ignores checkpoints)
|
|
395
|
+
qdrant-loader ingest --force
|
|
396
|
+
|
|
373
397
|
# Ingest specific project
|
|
374
398
|
qdrant-loader ingest --project my-project
|
|
375
399
|
|
|
@@ -397,9 +421,27 @@ async def ingest(
|
|
|
397
421
|
log_level,
|
|
398
422
|
profile,
|
|
399
423
|
force,
|
|
424
|
+
resume,
|
|
400
425
|
)
|
|
401
426
|
|
|
402
427
|
|
|
428
|
+
async def _start_webhook_server(
|
|
429
|
+
workspace: Path | None,
|
|
430
|
+
config: Path | None,
|
|
431
|
+
env: Path | None,
|
|
432
|
+
host: str,
|
|
433
|
+
port: int,
|
|
434
|
+
log_level: str,
|
|
435
|
+
) -> None:
|
|
436
|
+
"""Start webhook server for receiving connector events and triggering ingestion.
|
|
437
|
+
|
|
438
|
+
Internal function called by the future `serve` command. Not exposed as a user CLI command in v1.1.
|
|
439
|
+
"""
|
|
440
|
+
from qdrant_loader.cli.commands.webhook_cmd import run_webhook_command
|
|
441
|
+
|
|
442
|
+
await run_webhook_command(workspace, config, env, host, port, log_level)
|
|
443
|
+
|
|
444
|
+
|
|
403
445
|
@cli.command()
|
|
404
446
|
@option(
|
|
405
447
|
"--output-dir",
|
|
@@ -10,10 +10,25 @@ def __getattr__(name: str):
|
|
|
10
10
|
from .ingest import run_pipeline_ingestion
|
|
11
11
|
|
|
12
12
|
return run_pipeline_ingestion
|
|
13
|
+
if name == "serve_cmd":
|
|
14
|
+
from .serve_cmd import serve_cmd
|
|
15
|
+
|
|
16
|
+
return serve_cmd
|
|
17
|
+
if name == "jobs_cmd":
|
|
18
|
+
from .jobs_cmd import jobs_cmd
|
|
19
|
+
|
|
20
|
+
return jobs_cmd
|
|
21
|
+
if name == "run_webhook_command":
|
|
22
|
+
from .webhook_cmd import run_webhook_command
|
|
23
|
+
|
|
24
|
+
return run_webhook_command
|
|
13
25
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
14
26
|
|
|
15
27
|
|
|
16
28
|
__all__ = [
|
|
17
29
|
"run_init",
|
|
18
30
|
"run_pipeline_ingestion",
|
|
31
|
+
"serve_cmd",
|
|
32
|
+
"jobs_cmd",
|
|
33
|
+
"run_webhook_command",
|
|
19
34
|
]
|
|
@@ -40,7 +40,9 @@ def run_show_config(
|
|
|
40
40
|
|
|
41
41
|
# Setup/reconfigure logging once with workspace support
|
|
42
42
|
log_file = (
|
|
43
|
-
str(workspace_config.logs_path
|
|
43
|
+
str(workspace_config.logs_path / "config.log")
|
|
44
|
+
if workspace_config
|
|
45
|
+
else "qdrant-loader.log"
|
|
44
46
|
)
|
|
45
47
|
if getattr(LoggingConfig, "reconfigure", None): # Core supports reconfigure
|
|
46
48
|
if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined]
|
|
@@ -18,7 +18,9 @@ def run_config_command(
|
|
|
18
18
|
# Maintain test expectation: log via workspace-logger as before
|
|
19
19
|
workspace_config = _setup_workspace_impl(workspace) if workspace else None
|
|
20
20
|
log_file = (
|
|
21
|
-
str(workspace_config.logs_path
|
|
21
|
+
str(workspace_config.logs_path / "config.log")
|
|
22
|
+
if workspace_config
|
|
23
|
+
else "qdrant-loader.log"
|
|
22
24
|
)
|
|
23
25
|
if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined]
|
|
24
26
|
if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined]
|
|
@@ -15,6 +15,7 @@ async def run_pipeline_ingestion(
|
|
|
15
15
|
source_type: str | None,
|
|
16
16
|
source: str | None,
|
|
17
17
|
force: bool,
|
|
18
|
+
resume: bool = True,
|
|
18
19
|
metrics_dir: str | None = None,
|
|
19
20
|
) -> None:
|
|
20
21
|
from qdrant_loader.core.async_ingestion_pipeline import AsyncIngestionPipeline
|
|
@@ -32,6 +33,7 @@ async def run_pipeline_ingestion(
|
|
|
32
33
|
source_type=source_type,
|
|
33
34
|
source=source,
|
|
34
35
|
force=force,
|
|
36
|
+
resume=resume,
|
|
35
37
|
)
|
|
36
38
|
except Exception as e:
|
|
37
39
|
ingestion_error = e
|
|
@@ -36,8 +36,9 @@ async def run_ingest_command(
|
|
|
36
36
|
log_level: str,
|
|
37
37
|
profile: bool,
|
|
38
38
|
force: bool,
|
|
39
|
+
resume: bool = True,
|
|
39
40
|
) -> None:
|
|
40
|
-
"""Implementation for the `ingest` CLI command with
|
|
41
|
+
"""Implementation for the `ingest` CLI command with checkpoint resume support (WS-2)."""
|
|
41
42
|
|
|
42
43
|
ingest_start_time = time.perf_counter()
|
|
43
44
|
|
|
@@ -52,7 +53,9 @@ async def run_ingest_command(
|
|
|
52
53
|
|
|
53
54
|
# Setup/reconfigure logging with workspace support
|
|
54
55
|
log_file = (
|
|
55
|
-
str(workspace_config.logs_path
|
|
56
|
+
str(workspace_config.logs_path / "ingest.log")
|
|
57
|
+
if workspace_config
|
|
58
|
+
else "qdrant-loader.log"
|
|
56
59
|
)
|
|
57
60
|
if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined]
|
|
58
61
|
if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined]
|
|
@@ -87,6 +90,7 @@ async def run_ingest_command(
|
|
|
87
90
|
source_type=source_type,
|
|
88
91
|
source=source,
|
|
89
92
|
force=force,
|
|
93
|
+
resume=resume,
|
|
90
94
|
metrics_dir=(
|
|
91
95
|
str(workspace_config.metrics_path) if workspace_config else None
|
|
92
96
|
),
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""
|
|
2
|
+
qdrant-loader jobs admin CLI commands.
|
|
3
|
+
|
|
4
|
+
Subcommands: list [--status], retry <id>, trigger --source-type --source --mode, cancel <id>.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
from click.exceptions import ClickException
|
|
15
|
+
from click.types import Choice
|
|
16
|
+
from click.types import Path as ClickPath
|
|
17
|
+
|
|
18
|
+
from qdrant_loader.core.worker.job_types import JobType
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _init_queue(workspace: Path | None, config: Path | None, env: Path | None):
|
|
22
|
+
"""Load config, build StateManager. Returns state_manager."""
|
|
23
|
+
from qdrant_loader.cli.config_loader import (
|
|
24
|
+
load_config_with_workspace,
|
|
25
|
+
setup_workspace,
|
|
26
|
+
)
|
|
27
|
+
from qdrant_loader.config import get_global_config
|
|
28
|
+
from qdrant_loader.config.workspace import validate_workspace_flags
|
|
29
|
+
from qdrant_loader.core.state.state_manager import StateManager
|
|
30
|
+
|
|
31
|
+
# Validate mutually exclusive flags early
|
|
32
|
+
try:
|
|
33
|
+
validate_workspace_flags(workspace, config, env)
|
|
34
|
+
except ValueError as e:
|
|
35
|
+
raise ClickException(str(e))
|
|
36
|
+
|
|
37
|
+
if workspace:
|
|
38
|
+
ws_config = setup_workspace(workspace)
|
|
39
|
+
load_config_with_workspace(workspace_config=ws_config, skip_validation=True)
|
|
40
|
+
else:
|
|
41
|
+
resolved_config = config
|
|
42
|
+
if resolved_config is None:
|
|
43
|
+
default_config = Path("config.yaml")
|
|
44
|
+
resolved_config = default_config if default_config.exists() else None
|
|
45
|
+
if resolved_config is None:
|
|
46
|
+
raise ClickException("No config found. Use --workspace or --config.")
|
|
47
|
+
load_config_with_workspace(
|
|
48
|
+
workspace_config=None,
|
|
49
|
+
config_path=resolved_config,
|
|
50
|
+
env_path=env,
|
|
51
|
+
skip_validation=True,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
global_config = get_global_config()
|
|
55
|
+
state_manager = StateManager(global_config.state_management)
|
|
56
|
+
return state_manager
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
async def _run_with_queue(workspace, config, env, coro_factory):
|
|
60
|
+
"""Initialize queue, run coro_factory(queue), dispose state manager."""
|
|
61
|
+
state_manager = _init_queue(workspace, config, env)
|
|
62
|
+
await state_manager.initialize()
|
|
63
|
+
queue_instance = None
|
|
64
|
+
try:
|
|
65
|
+
from qdrant_loader.core.worker.queue import SQLiteJobQueue
|
|
66
|
+
|
|
67
|
+
queue_instance = SQLiteJobQueue(
|
|
68
|
+
state_manager.session_factory,
|
|
69
|
+
db_op_lock=state_manager.queue_db_op_lock,
|
|
70
|
+
)
|
|
71
|
+
return await coro_factory(queue_instance)
|
|
72
|
+
finally:
|
|
73
|
+
await state_manager.dispose()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# ────────────────────────────────────────────────
|
|
77
|
+
# Click group
|
|
78
|
+
# ────────────────────────────────────────────────
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@click.group("jobs")
|
|
82
|
+
def jobs_cmd():
|
|
83
|
+
"""Inspect and manage background ingestion jobs."""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _common_options(fn):
|
|
87
|
+
"""Decorator that adds --workspace, --config, --env options."""
|
|
88
|
+
fn = click.option(
|
|
89
|
+
"--workspace",
|
|
90
|
+
type=ClickPath(path_type=Path),
|
|
91
|
+
default=None,
|
|
92
|
+
help="Workspace directory.",
|
|
93
|
+
)(fn)
|
|
94
|
+
fn = click.option(
|
|
95
|
+
"--config",
|
|
96
|
+
"config_path",
|
|
97
|
+
type=ClickPath(path_type=Path),
|
|
98
|
+
default=None,
|
|
99
|
+
help="Path to config.yaml.",
|
|
100
|
+
)(fn)
|
|
101
|
+
fn = click.option(
|
|
102
|
+
"--env",
|
|
103
|
+
"env_path",
|
|
104
|
+
type=ClickPath(path_type=Path),
|
|
105
|
+
default=None,
|
|
106
|
+
help="Path to .env file.",
|
|
107
|
+
)(fn)
|
|
108
|
+
return fn
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ────────────────────────────────────────────────
|
|
112
|
+
# jobs list
|
|
113
|
+
# ────────────────────────────────────────────────
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@jobs_cmd.command("list")
|
|
117
|
+
@click.option(
|
|
118
|
+
"--status",
|
|
119
|
+
type=Choice(
|
|
120
|
+
["pending", "running", "done", "failed", "cancelled"], case_sensitive=False
|
|
121
|
+
),
|
|
122
|
+
default=None,
|
|
123
|
+
help="Filter by job status.",
|
|
124
|
+
)
|
|
125
|
+
@click.option(
|
|
126
|
+
"--limit",
|
|
127
|
+
type=click.IntRange(min=1, max=1000),
|
|
128
|
+
default=50,
|
|
129
|
+
show_default=True,
|
|
130
|
+
help="Max rows to return.",
|
|
131
|
+
)
|
|
132
|
+
@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
|
|
133
|
+
@_common_options
|
|
134
|
+
def jobs_list(status, limit, output_json, workspace, config_path, env_path):
|
|
135
|
+
"""List jobs, optionally filtered by status."""
|
|
136
|
+
|
|
137
|
+
async def _list(queue):
|
|
138
|
+
jobs = await queue.list(status=status, limit=limit)
|
|
139
|
+
if output_json:
|
|
140
|
+
rows = []
|
|
141
|
+
for j in jobs:
|
|
142
|
+
rows.append(
|
|
143
|
+
{
|
|
144
|
+
"id": j.id,
|
|
145
|
+
"type": j.type,
|
|
146
|
+
"status": j.status,
|
|
147
|
+
"attempts": j.attempts,
|
|
148
|
+
"enqueued_at": (
|
|
149
|
+
j.enqueued_at.isoformat() if j.enqueued_at else None
|
|
150
|
+
),
|
|
151
|
+
"started_at": (
|
|
152
|
+
j.started_at.isoformat() if j.started_at else None
|
|
153
|
+
),
|
|
154
|
+
"finished_at": (
|
|
155
|
+
j.finished_at.isoformat() if j.finished_at else None
|
|
156
|
+
),
|
|
157
|
+
"last_error": j.last_error,
|
|
158
|
+
"payload": json.loads(j.payload_json) if j.payload_json else {},
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
click.echo(json.dumps(rows, indent=2))
|
|
162
|
+
else:
|
|
163
|
+
if not jobs:
|
|
164
|
+
click.echo("No jobs found.")
|
|
165
|
+
return
|
|
166
|
+
header = f"{'ID':>6} {'STATUS':<10} {'TYPE':<20} {'ATTEMPTS':>8} {'ENQUEUED_AT':<27} LAST_ERROR"
|
|
167
|
+
click.echo(header)
|
|
168
|
+
click.echo("-" * len(header))
|
|
169
|
+
for j in jobs:
|
|
170
|
+
enq = j.enqueued_at.isoformat() if j.enqueued_at else ""
|
|
171
|
+
err = (j.last_error or "")[:40]
|
|
172
|
+
click.echo(
|
|
173
|
+
f"{j.id:>6} {j.status:<10} {j.type:<20} {j.attempts:>8} {enq:<27} {err}"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
asyncio.run(_run_with_queue(workspace, config_path, env_path, _list))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# ────────────────────────────────────────────────
|
|
180
|
+
# jobs retry
|
|
181
|
+
# ────────────────────────────────────────────────
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
@jobs_cmd.command("retry")
|
|
185
|
+
@click.argument("job_id", type=int)
|
|
186
|
+
@_common_options
|
|
187
|
+
def jobs_retry(job_id, workspace, config_path, env_path):
|
|
188
|
+
"""Reset a failed or done job back to pending."""
|
|
189
|
+
|
|
190
|
+
async def _retry(queue):
|
|
191
|
+
ok = await queue.reset_to_pending(job_id)
|
|
192
|
+
if ok:
|
|
193
|
+
click.echo(f"Job {job_id} reset to pending.")
|
|
194
|
+
else:
|
|
195
|
+
raise ClickException(
|
|
196
|
+
f"Job {job_id} not found or is not in failed/done state."
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
asyncio.run(_run_with_queue(workspace, config_path, env_path, _retry))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
# ────────────────────────────────────────────────
|
|
203
|
+
# jobs trigger
|
|
204
|
+
# ────────────────────────────────────────────────
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@jobs_cmd.command("trigger")
|
|
208
|
+
@click.option(
|
|
209
|
+
"--source-type", required=True, help="Source type (e.g. git, confluence)."
|
|
210
|
+
)
|
|
211
|
+
@click.option("--source", required=True, help="Source name as configured.")
|
|
212
|
+
@click.option(
|
|
213
|
+
"--mode",
|
|
214
|
+
type=Choice(["bulk", "incremental"], case_sensitive=False),
|
|
215
|
+
required=True,
|
|
216
|
+
help="Ingestion mode.",
|
|
217
|
+
)
|
|
218
|
+
@click.option("--project", "project_id", required=True, help="Project ID.")
|
|
219
|
+
@_common_options
|
|
220
|
+
def jobs_trigger(
|
|
221
|
+
source_type, source, mode, project_id, workspace, config_path, env_path
|
|
222
|
+
):
|
|
223
|
+
"""Enqueue a new ingestion job immediately."""
|
|
224
|
+
|
|
225
|
+
async def _trigger(queue):
|
|
226
|
+
job_type = (
|
|
227
|
+
JobType.BULK_INGEST if mode.lower() == "bulk" else JobType.INCREMENTAL_PULL
|
|
228
|
+
)
|
|
229
|
+
payload = {
|
|
230
|
+
"project_id": project_id,
|
|
231
|
+
"source_type": source_type,
|
|
232
|
+
"source": source,
|
|
233
|
+
"source_lock": f"{project_id}:{source_type}:{source}",
|
|
234
|
+
}
|
|
235
|
+
job = await queue.enqueue(job_type, payload)
|
|
236
|
+
click.echo(f"Enqueued job {job.id} (type={job_type}, status={job.status}).")
|
|
237
|
+
|
|
238
|
+
asyncio.run(_run_with_queue(workspace, config_path, env_path, _trigger))
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
# ────────────────────────────────────────────────
|
|
242
|
+
# jobs cancel
|
|
243
|
+
# ────────────────────────────────────────────────
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
@jobs_cmd.command("cancel")
|
|
247
|
+
@click.argument("job_id", type=int)
|
|
248
|
+
@_common_options
|
|
249
|
+
def jobs_cancel(job_id, workspace, config_path, env_path):
|
|
250
|
+
"""Cancel a pending job."""
|
|
251
|
+
|
|
252
|
+
async def _cancel(queue):
|
|
253
|
+
ok = await queue.cancel(job_id)
|
|
254
|
+
if ok:
|
|
255
|
+
click.echo(f"Job {job_id} cancelled.")
|
|
256
|
+
else:
|
|
257
|
+
raise ClickException(f"Job {job_id} not found or is not in pending state.")
|
|
258
|
+
|
|
259
|
+
asyncio.run(_run_with_queue(workspace, config_path, env_path, _cancel))
|