qdrant-loader 1.0.2__tar.gz → 1.0.3__tar.gz

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qdrant-loader
-Version: 1.0.2
+Version: 1.0.3
 Summary: A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database.
 Author-email: Martin Papy <martin.papy@cbtw.tech>
 License-Expression: GPL-3.0
@@ -37,7 +37,7 @@ Requires-Dist: pydantic-settings>=2.0.0
 Requires-Dist: structlog>=23.0.0
 Requires-Dist: httpx>=0.24.0
 Requires-Dist: openai>=1.0.0
-Requires-Dist: qdrant-loader-core[openai]==1.0.2
+Requires-Dist: qdrant-loader-core[openai]==1.0.3
 Requires-Dist: qdrant-client>=1.10.0
 Requires-Dist: PyYAML>=6.0.0
 Requires-Dist: beautifulsoup4>=4.12.0
@@ -48,7 +48,7 @@ Requires-Dist: langchain-core>=0.3.0
 Requires-Dist: langchain-community>=0.0.38
 Requires-Dist: langchain-text-splitters>=0.3.9
 Requires-Dist: numpy<2.0.0,>=1.26.0
-Requires-Dist: GitPython>=3.1.40
+Requires-Dist: GitPython>=3.1.50
 Requires-Dist: atlassian-python-api>=3.41.0
 Requires-Dist: tqdm>=4.66.0
 Requires-Dist: SQLAlchemy>=2.0.0
@@ -65,10 +65,16 @@ Requires-Dist: psutil>=5.9.0
 Requires-Dist: tree-sitter-languages>=1.10.0
 Requires-Dist: tree-sitter<0.21
 Requires-Dist: markitdown[all]>=0.1.3
+Requires-Dist: networkx>=3.0.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: questionary>=2.0.0
 Requires-Dist: packaging>=21.0
 Requires-Dist: prometheus-client<1.0.0,>=0.19.0
+Provides-Extra: server
+Requires-Dist: fastapi>=0.104.0; extra == "server"
+Requires-Dist: uvicorn[standard]>=0.24.0; sys_platform != "win32" and extra == "server"
+Requires-Dist: uvicorn>=0.24.0; sys_platform == "win32" and extra == "server"
+Requires-Dist: PyJWT[crypto]>=2.8.0; extra == "server"
 
 # QDrant Loader
 

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "qdrant-loader"
-version = "1.0.2"
+version = "1.0.3"
 description = "A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database."
 readme = "README.md"
 requires-python = ">=3.12"
@@ -35,7 +35,7 @@ dependencies = [
     "structlog>=23.0.0",
     "httpx>=0.24.0",
     "openai>=1.0.0",
-    "qdrant-loader-core[openai]==1.0.2",
+    "qdrant-loader-core[openai]==1.0.3",
     "qdrant-client>=1.10.0",
     "PyYAML>=6.0.0",
     "beautifulsoup4>=4.12.0",
@@ -46,7 +46,7 @@ dependencies = [
     "langchain-community>=0.0.38",
     "langchain-text-splitters>=0.3.9",
     "numpy>=1.26.0,<2.0.0",
-    "GitPython>=3.1.40",
+    "GitPython>=3.1.50",
     "atlassian-python-api>=3.41.0",
     "tqdm>=4.66.0",
     "SQLAlchemy>=2.0.0",
@@ -63,6 +63,7 @@ dependencies = [
     "tree-sitter-languages>=1.10.0",
     "tree-sitter<0.21",
     "markitdown[all]>=0.1.3",
+    "networkx>=3.0.0",
     "rich>=13.0.0",
     "questionary>=2.0.0",
     "packaging>=21.0",
@@ -87,6 +88,14 @@ classifiers = [
     "Typing :: Typed",
 ]
 
+[project.optional-dependencies]
+server = [
+    "fastapi>=0.104.0",
+    "uvicorn[standard]>=0.24.0 ; sys_platform != 'win32'",
+    "uvicorn>=0.24.0 ; sys_platform == 'win32'",
+    "PyJWT[crypto]>=2.8.0",
+]
+
 [project.urls]
 Homepage = "https://qdrant-loader.net"
 Documentation = "https://qdrant-loader.net/docs/packages/qdrant-loader/README.html"

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/__init__.py

@@ -43,7 +43,14 @@ def __getattr__(name):
 
         return QdrantManager
     else:
-        raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+        try:
+            import importlib
+
+            return importlib.import_module(f"{__name__}.{name}")
+        except ImportError as err:
+            raise AttributeError(
+                f"module '{__name__}' has no attribute '{name}'"
+            ) from err
 
 
 __all__ = [

qdrant_loader-1.0.3/src/qdrant_loader/cli/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from qdrant_loader.cli.cli import cli
+
+if __name__ == "__main__":
+    cli()

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/cli/cli.py

@@ -11,6 +11,8 @@ from click.utils import echo
 
 # async_command is needed at module level for @async_command decorator on commands.
 from qdrant_loader.cli.asyncio import async_command  # noqa: F401
+from qdrant_loader.cli.commands.jobs_cmd import jobs_cmd
+from qdrant_loader.cli.commands.serve_cmd import serve_cmd as serve_command
 from qdrant_loader.utils.sensitive import sanitize_exception_message
 
 # Heavy modules are lazy-imported to keep CLI startup fast.
@@ -77,7 +79,9 @@ def _setup_logging(log_level: str, workspace_config=None) -> None:
 
         log_format = "console"
         log_file = (
-            str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
+            str(workspace_config.logs_path / "cli.log")
+            if workspace_config
+            else "qdrant-loader.log"
         )
         LoggingConfig.setup(level=log_level, format=log_format, file=log_file)
         # update module-global logger
@@ -96,9 +100,13 @@ def _check_for_updates() -> None:
         pass
 
 
-def _setup_workspace(workspace_path: Path):
-    workspace_config = _setup_workspace_impl(workspace_path)
-    return workspace_config
+def _setup_workspace(workspace_path):
+    try:
+        return _setup_workspace_impl(workspace_path)
+    except ValueError as e:
+        raise ClickException(str(e)) from e
+    except Exception as e:  # pragma: no cover - handled by CLI tests
+        raise ClickException(f"Failed to setup workspace: {str(e)!s}") from e
 
 
 @group(name="qdrant-loader")
@@ -120,6 +128,10 @@ def cli(log_level: str = "INFO") -> None:
     _check_for_updates()
 
 
+cli.add_command(serve_command)
+cli.add_command(jobs_cmd)
+
+
 def _create_database_directory(path: Path) -> bool:
     """Create database directory with user confirmation.
 
@@ -400,6 +412,23 @@ async def ingest(
     )
 
 
+async def _start_webhook_server(
+    workspace: Path | None,
+    config: Path | None,
+    env: Path | None,
+    host: str,
+    port: int,
+    log_level: str,
+) -> None:
+    """Start webhook server for receiving connector events and triggering ingestion.
+
+    Internal function called by the future `serve` command. Not exposed as a user CLI command in v1.1.
+    """
+    from qdrant_loader.cli.commands.webhook_cmd import run_webhook_command
+
+    await run_webhook_command(workspace, config, env, host, port, log_level)
+
+
 @cli.command()
 @option(
     "--output-dir",

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/cli/commands/__init__.py

@@ -10,10 +10,25 @@ def __getattr__(name: str):
         from .ingest import run_pipeline_ingestion
 
         return run_pipeline_ingestion
+    if name == "serve_cmd":
+        from .serve_cmd import serve_cmd
+
+        return serve_cmd
+    if name == "jobs_cmd":
+        from .jobs_cmd import jobs_cmd
+
+        return jobs_cmd
+    if name == "run_webhook_command":
+        from .webhook_cmd import run_webhook_command
+
+        return run_webhook_command
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
 __all__ = [
     "run_init",
     "run_pipeline_ingestion",
+    "serve_cmd",
+    "jobs_cmd",
+    "run_webhook_command",
 ]

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/cli/commands/config.py

@@ -40,7 +40,9 @@ def run_show_config(
 
         # Setup/reconfigure logging once with workspace support
         log_file = (
-            str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
+            str(workspace_config.logs_path / "config.log")
+            if workspace_config
+            else "qdrant-loader.log"
         )
         if getattr(LoggingConfig, "reconfigure", None):  # Core supports reconfigure
             if getattr(LoggingConfig, "_initialized", False):  # type: ignore[attr-defined]

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/cli/commands/config_cmd.py

@@ -18,7 +18,9 @@ def run_config_command(
         # Maintain test expectation: log via workspace-logger as before
         workspace_config = _setup_workspace_impl(workspace) if workspace else None
         log_file = (
-            str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
+            str(workspace_config.logs_path / "config.log")
+            if workspace_config
+            else "qdrant-loader.log"
         )
         if getattr(LoggingConfig, "reconfigure", None):  # type: ignore[attr-defined]
             if getattr(LoggingConfig, "_initialized", False):  # type: ignore[attr-defined]

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/cli/commands/ingest_cmd.py

@@ -52,7 +52,9 @@ async def run_ingest_command(
 
         # Setup/reconfigure logging with workspace support
         log_file = (
-            str(workspace_config.logs_path) if workspace_config else "qdrant-loader.log"
+            str(workspace_config.logs_path / "ingest.log")
+            if workspace_config
+            else "qdrant-loader.log"
         )
         if getattr(LoggingConfig, "reconfigure", None):  # type: ignore[attr-defined]
             if getattr(LoggingConfig, "_initialized", False):  # type: ignore[attr-defined]

{qdrant_loader-1.0.2 → qdrant_loader-1.0.3}/src/qdrant_loader/cli/commands/init_cmd.py

@@ -83,7 +83,7 @@ async def run_init_command(
 
             # Setup logging again with workspace-aware file path
             log_file = (
-                str(workspace_config.logs_path)
+                str(workspace_config.logs_path / "init.log")
                 if workspace_config
                 else "qdrant-loader.log"
             )

qdrant_loader-1.0.3/src/qdrant_loader/cli/commands/jobs_cmd.py

@@ -0,0 +1,256 @@
+"""
+qdrant-loader jobs admin CLI commands.
+
+Subcommands: list [--status], retry <id>, trigger --source-type --source --mode, cancel <id>.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+
+import click
+from click.exceptions import ClickException
+from click.types import Choice
+from click.types import Path as ClickPath
+
+from qdrant_loader.core.worker.job_types import JobType
+
+
+def _init_queue(workspace: Path | None, config: Path | None, env: Path | None):
+    """Load config, build StateManager. Returns state_manager."""
+    from qdrant_loader.cli.config_loader import (
+        load_config_with_workspace,
+        setup_workspace,
+    )
+    from qdrant_loader.config import get_global_config
+    from qdrant_loader.config.workspace import validate_workspace_flags
+    from qdrant_loader.core.state.state_manager import StateManager
+
+    # Validate mutually exclusive flags early
+    try:
+        validate_workspace_flags(workspace, config, env)
+    except ValueError as e:
+        raise ClickException(str(e))
+
+    if workspace:
+        ws_config = setup_workspace(workspace)
+        load_config_with_workspace(workspace_config=ws_config, skip_validation=True)
+    else:
+        resolved_config = config
+        if resolved_config is None:
+            default_config = Path("config.yaml")
+            resolved_config = default_config if default_config.exists() else None
+        if resolved_config is None:
+            raise ClickException("No config found. Use --workspace or --config.")
+        load_config_with_workspace(
+            workspace_config=None,
+            config_path=resolved_config,
+            env_path=env,
+            skip_validation=True,
+        )
+
+    global_config = get_global_config()
+    state_manager = StateManager(global_config.state_management)
+    return state_manager
+
+
+async def _run_with_queue(workspace, config, env, coro_factory):
+    """Initialize queue, run coro_factory(queue), dispose state manager."""
+    state_manager = _init_queue(workspace, config, env)
+    await state_manager.initialize()
+    queue_instance = None
+    try:
+        from qdrant_loader.core.worker.queue import SQLiteJobQueue
+
+        queue_instance = SQLiteJobQueue(state_manager.session_factory)
+        return await coro_factory(queue_instance)
+    finally:
+        await state_manager.dispose()
+
+
+# ────────────────────────────────────────────────
+# Click group
+# ────────────────────────────────────────────────
+
+
+@click.group("jobs")
+def jobs_cmd():
+    """Inspect and manage background ingestion jobs."""
+
+
+def _common_options(fn):
+    """Decorator that adds --workspace, --config, --env options."""
+    fn = click.option(
+        "--workspace",
+        type=ClickPath(path_type=Path),
+        default=None,
+        help="Workspace directory.",
+    )(fn)
+    fn = click.option(
+        "--config",
+        "config_path",
+        type=ClickPath(path_type=Path),
+        default=None,
+        help="Path to config.yaml.",
+    )(fn)
+    fn = click.option(
+        "--env",
+        "env_path",
+        type=ClickPath(path_type=Path),
+        default=None,
+        help="Path to .env file.",
+    )(fn)
+    return fn
+
+
+# ────────────────────────────────────────────────
+# jobs list
+# ────────────────────────────────────────────────
+
+
+@jobs_cmd.command("list")
+@click.option(
+    "--status",
+    type=Choice(
+        ["pending", "running", "done", "failed", "cancelled"], case_sensitive=False
+    ),
+    default=None,
+    help="Filter by job status.",
+)
+@click.option(
+    "--limit",
+    type=click.IntRange(min=1, max=1000),
+    default=50,
+    show_default=True,
+    help="Max rows to return.",
+)
+@click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
+@_common_options
+def jobs_list(status, limit, output_json, workspace, config_path, env_path):
+    """List jobs, optionally filtered by status."""
+
+    async def _list(queue):
+        jobs = await queue.list(status=status, limit=limit)
+        if output_json:
+            rows = []
+            for j in jobs:
+                rows.append(
+                    {
+                        "id": j.id,
+                        "type": j.type,
+                        "status": j.status,
+                        "attempts": j.attempts,
+                        "enqueued_at": (
+                            j.enqueued_at.isoformat() if j.enqueued_at else None
+                        ),
+                        "started_at": (
+                            j.started_at.isoformat() if j.started_at else None
+                        ),
+                        "finished_at": (
+                            j.finished_at.isoformat() if j.finished_at else None
+                        ),
+                        "last_error": j.last_error,
+                        "payload": json.loads(j.payload_json) if j.payload_json else {},
+                    }
+                )
+            click.echo(json.dumps(rows, indent=2))
+        else:
+            if not jobs:
+                click.echo("No jobs found.")
+                return
+            header = f"{'ID':>6}  {'STATUS':<10}  {'TYPE':<20}  {'ATTEMPTS':>8}  {'ENQUEUED_AT':<27}  LAST_ERROR"
+            click.echo(header)
+            click.echo("-" * len(header))
+            for j in jobs:
+                enq = j.enqueued_at.isoformat() if j.enqueued_at else ""
+                err = (j.last_error or "")[:40]
+                click.echo(
+                    f"{j.id:>6}  {j.status:<10}  {j.type:<20}  {j.attempts:>8}  {enq:<27}  {err}"
+                )
+
+    asyncio.run(_run_with_queue(workspace, config_path, env_path, _list))
+
+
+# ────────────────────────────────────────────────
+# jobs retry
+# ────────────────────────────────────────────────
+
+
+@jobs_cmd.command("retry")
+@click.argument("job_id", type=int)
+@_common_options
+def jobs_retry(job_id, workspace, config_path, env_path):
+    """Reset a failed or done job back to pending."""
+
+    async def _retry(queue):
+        ok = await queue.reset_to_pending(job_id)
+        if ok:
+            click.echo(f"Job {job_id} reset to pending.")
+        else:
+            raise ClickException(
+                f"Job {job_id} not found or is not in failed/done state."
+            )
+
+    asyncio.run(_run_with_queue(workspace, config_path, env_path, _retry))
+
+
+# ────────────────────────────────────────────────
+# jobs trigger
+# ────────────────────────────────────────────────
+
+
+@jobs_cmd.command("trigger")
+@click.option(
+    "--source-type", required=True, help="Source type (e.g. git, confluence)."
+)
+@click.option("--source", required=True, help="Source name as configured.")
+@click.option(
+    "--mode",
+    type=Choice(["bulk", "incremental"], case_sensitive=False),
+    required=True,
+    help="Ingestion mode.",
+)
+@click.option("--project", "project_id", required=True, help="Project ID.")
+@_common_options
+def jobs_trigger(
+    source_type, source, mode, project_id, workspace, config_path, env_path
+):
+    """Enqueue a new ingestion job immediately."""
+
+    async def _trigger(queue):
+        job_type = (
+            JobType.BULK_INGEST if mode.lower() == "bulk" else JobType.INCREMENTAL_PULL
+        )
+        payload = {
+            "project_id": project_id,
+            "source_type": source_type,
+            "source": source,
+            "source_lock": f"{project_id}:{source_type}:{source}",
+        }
+        job = await queue.enqueue(job_type, payload)
+        click.echo(f"Enqueued job {job.id} (type={job_type}, status={job.status}).")
+
+    asyncio.run(_run_with_queue(workspace, config_path, env_path, _trigger))
+
+
+# ────────────────────────────────────────────────
+# jobs cancel
+# ────────────────────────────────────────────────
+
+
+@jobs_cmd.command("cancel")
+@click.argument("job_id", type=int)
+@_common_options
+def jobs_cancel(job_id, workspace, config_path, env_path):
+    """Cancel a pending job."""
+
+    async def _cancel(queue):
+        ok = await queue.cancel(job_id)
+        if ok:
+            click.echo(f"Job {job_id} cancelled.")
+        else:
+            raise ClickException(f"Job {job_id} not found or is not in pending state.")
+
+    asyncio.run(_run_with_queue(workspace, config_path, env_path, _cancel))