warren 0.1.1__py3-none-any.whl

runtime_scripts/lib/cli.py

@@ -0,0 +1,30 @@
+"""
+Common CLI argument helpers for launcher scripts.
+"""
+
+import argparse
+from pathlib import Path
+
+
+def add_common_args(parser: argparse.ArgumentParser) -> None:
+    """Add arguments shared across all launcher scripts.
+
+    Adds: --config-file, --worker-name, --debug.
+    """
+    parser.add_argument(
+        "--config-file",
+        type=Path,
+        default=None,
+        help=("Path to RuntimeConfig YAML. Default: ./pipeline/config.yaml"),
+    )
+    parser.add_argument(
+        "--worker-name",
+        default=None,
+        help="Unique worker name (default: auto-generated).",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        default=False,
+        help="Enable DEBUG logging (default: INFO).",
+    )

runtime_scripts/lib/logging_setup.py

@@ -0,0 +1,30 @@
+"""
+Logging setup for runtime script entry points.
+
+Workaround for basics.logging auto-configuring the root logger at
+DEBUG on import. Call ``configure_logging()`` early in ``main()``
+before any substantive log output.
+"""
+
+import logging
+
+
+def resolve_log_level(*, debug: bool) -> int:
+    """Return the logging level based on the --debug flag.
+
+    :param debug: whether debug logging is enabled.
+    :return: ``logging.DEBUG`` or ``logging.INFO``.
+    """
+    return logging.DEBUG if debug else logging.INFO
+
+
+def configure_logging(*, debug: bool) -> None:
+    """Configure logging for a CLI entry point.
+
+    :param debug: whether debug logging is enabled.
+    """
+    from basics.logging import setup_logging
+
+    log_level = resolve_log_level(debug=debug)
+    setup_logging(log_level=log_level)
+    logging.getLogger().setLevel(log_level)

runtime_scripts/lib/pipeline.py

@@ -0,0 +1,178 @@
+"""
+Pipeline spec and config loading utilities.
+
+Shared by all launcher scripts for resolving pipeline specs from
+directories, files, or dotted module paths.
+"""
+
+import importlib
+import importlib.util
+import logging
+from pathlib import Path
+
+from basics.logging import get_logger
+
+from warren.runtime.config import RuntimeConfig
+from warren.runtime.spec import PipelineSpec
+
+
+module_logger: logging.Logger = get_logger(__name__)
+
+DEFAULT_PIPELINE_DIR: str = "./pipeline"
+DEFAULT_SPEC_MODULE: str = "pipeline_spec"
+DEFAULT_SPEC_VAR: str = "PIPELINE"
+DEFAULT_CONFIG_FILE: str = "config.yaml"
+
+
+def split_var_suffix(spec_str: str) -> tuple[str, str]:
+    """Split ``path_or_module:VAR`` into ``(path_or_module, var_name)``.
+
+    :return: tuple of (location, variable name).
+    """
+    if ":" in spec_str:
+        location, var_name = spec_str.rsplit(":", 1)
+        return location, var_name
+    return spec_str, DEFAULT_SPEC_VAR
+
+
+def import_file(file_path: Path) -> object:
+    """Import a .py file as a module and return it.
+
+    The spec module's own import/syntax/runtime errors propagate
+    unchanged — the calling launcher wraps them with phase context
+    (e.g. "Unable to load pipeline spec from: ..."). Only failure to
+    create the module spec raises ``ImportError`` here.
+    """
+    spec = importlib.util.spec_from_file_location(
+        file_path.stem,
+        file_path,
+    )
+    if spec is None or spec.loader is None:
+        msg = f"Cannot create module spec from {file_path}"
+        raise ImportError(msg)
+
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def resolve_pipeline_file(directory: Path) -> Path:
+    """Find ``pipeline_spec.py`` inside a directory (non-recursive).
+
+    :return: path to the spec file.
+    :raises FileNotFoundError: if no pipeline_spec.py exists in the
+        directory.
+    """
+    candidate = directory / f"{DEFAULT_SPEC_MODULE}.py"
+    if candidate.is_file():
+        return candidate
+    msg = f"No {DEFAULT_SPEC_MODULE}.py in directory: {directory}"
+    raise FileNotFoundError(msg)
+
+
+def load_pipeline(
+    spec_str: str,
+    logger: logging.Logger | None = None,
+) -> tuple[PipelineSpec, Path | None]:
+    """Resolve a pipeline spec from the given location string.
+
+    Resolution order:
+
+    1. Split off ``:VAR`` suffix (default: ``PIPELINE``).
+    2. If the location exists as a directory -> look for
+       ``pipeline_spec.py`` inside, import it.
+    3. If it exists as a file -> import it directly.
+    4. Otherwise -> try ``importlib.import_module(location)``.
+    5. Read ``VAR`` from the loaded module.
+
+    :return: tuple of (PipelineSpec, resolved_dir) where
+        ``resolved_dir`` is the directory the spec was found in
+        (None for dotted-module imports).
+    :raises FileNotFoundError: if directory/file does not exist or
+        contains no pipeline_spec.py.
+    :raises ImportError: if dotted module cannot be imported.
+    :raises AttributeError: if the variable is not found in the module.
+    :raises TypeError: if the variable is not a PipelineSpec.
+    """
+    log = logger or module_logger
+    location, var_name = split_var_suffix(spec_str)
+
+    resolved_dir: Path | None = None
+    path = Path(location)
+
+    if path.is_dir():
+        spec_file = resolve_pipeline_file(path)
+        log.info(
+            f"Loading pipeline spec from directory: {path} "
+            f"(file: {spec_file.name}, var: {var_name})"
+        )
+        mod = import_file(spec_file)
+        resolved_dir = path
+
+    elif path.is_file():
+        log.info(f"Loading pipeline spec from file: {path} (var: {var_name})")
+        mod = import_file(path)
+        resolved_dir = path.parent
+
+    else:
+        log.info(f"Loading pipeline spec from module: {location} (var: {var_name})")
+        mod = importlib.import_module(location)
+
+    module_name = getattr(mod, "__name__", location)
+
+    if not hasattr(mod, var_name):
+        msg = f"Module {module_name} has no attribute '{var_name}'"
+        raise AttributeError(msg)
+
+    pipeline = getattr(mod, var_name)
+
+    if not isinstance(pipeline, PipelineSpec):
+        msg = (
+            f"{var_name} in {module_name} is "
+            f"{type(pipeline).__name__}, expected PipelineSpec"
+        )
+        raise TypeError(msg)
+
+    return pipeline, resolved_dir
+
+
+def resolve_config_path(
+    config_file: Path | None,
+    pipeline_dir: Path | None,
+) -> Path:
+    """Determine the config file path.
+
+    :param config_file: explicit config path (takes precedence).
+    :param pipeline_dir: pipeline directory to look for config.yaml in.
+    :raises FileNotFoundError: if no config.yaml found in pipeline_dir.
+    :raises ValueError: if neither config_file nor pipeline_dir provided.
+    """
+    if config_file is not None:
+        return config_file
+
+    if pipeline_dir is not None:
+        candidate = pipeline_dir / DEFAULT_CONFIG_FILE
+        if candidate.is_file():
+            return candidate
+        msg = (
+            f"No {DEFAULT_CONFIG_FILE} in pipeline directory {pipeline_dir} "
+            f"and no --config-file specified"
+        )
+        raise FileNotFoundError(msg)
+
+    msg = (
+        "--config-file is required when pipeline spec is loaded "
+        "from a Python module (no directory to infer config.yaml from)"
+    )
+    raise ValueError(msg)
+
+
+def load_config(config_file: Path) -> RuntimeConfig:
+    """Load a RuntimeConfig from a YAML file.
+
+    :raises FileNotFoundError: if the file does not exist.
+    """
+    if not config_file.is_file():
+        msg = f"Config file not found: {config_file}"
+        raise FileNotFoundError(msg)
+    return RuntimeConfig.from_yaml(config_file)

runtime_scripts/lib/runner.py

@@ -0,0 +1,86 @@
+"""
+Shared worker lifecycle for all launcher scripts.
+
+Provides the single ``run()`` function that all launchers call.
+The only thing that differs between launchers is the
+``runner_factory_func`` that creates the runner instance.
+"""
+
+import logging
+import uuid
+from collections.abc import Callable
+from pathlib import Path
+
+from basics.logging import get_logger
+
+from runtime_scripts.lib.pipeline import (
+    load_config,
+)
+from warren.exceptions import WarrenError
+from warren.runtime.config import RuntimeConfig
+from warren.workers.runners import WorkerRunnerBase
+
+
+module_logger: logging.Logger = get_logger(__name__)
+
+RunnerFactoryFunc = Callable[[RuntimeConfig, str], WorkerRunnerBase]
+
+
+async def run(
+    *,
+    runner_factory_func: RunnerFactoryFunc,
+    config_file: Path,
+    worker_name: str | None = None,
+    worker_name_prefix: str = "worker",
+    debug: bool = False,
+    logger: logging.Logger | None = None,
+) -> None:
+    """Load config, create a runner via factory, and run it.
+
+    This is the shared entry point for all launcher scripts. Each
+    launcher provides its own ``runner_factory_func`` that creates the
+    appropriate runner from ``(config, worker_name)``.
+
+    :param runner_factory_func: callable that takes
+        ``(config: RuntimeConfig, worker_name: str)`` and returns a
+        ``WorkerRunnerBase`` instance. Use ``partial()`` to bind
+        additional arguments (e.g. ``worker_spec``).
+    :param config_file: path to RuntimeConfig YAML.
+    :param worker_name: unique worker instance name. Defaults to
+        ``<worker_name_prefix>-<uuid8>``.
+    :param worker_name_prefix: prefix for auto-generated worker names.
+    :param debug: enable DEBUG logging.
+    :param logger: optional logger override.
+    """
+    log = logger or module_logger
+
+    try:
+        config = load_config(config_file)
+    except Exception as e:
+        msg = f"Unable to load config from: {config_file}"
+        raise WarrenError(msg) from e
+
+    log.info(f"Loaded config from: {config_file}")
+
+    resolved_worker_name = worker_name or f"{worker_name_prefix}-{uuid.uuid4().hex[:8]}"
+
+    try:
+        runner = runner_factory_func(config, resolved_worker_name)
+    except Exception as e:
+        msg = f"Unable to create runner for: {resolved_worker_name}"
+        raise WarrenError(msg) from e
+
+    try:
+        try:
+            await runner.setup()
+        except Exception as e:
+            msg = f"Worker setup failed for: {resolved_worker_name}"
+            raise WarrenError(msg) from e
+
+        try:
+            await runner.run()
+        except Exception as e:
+            msg = f"Worker run failed for: {resolved_worker_name}"
+            raise WarrenError(msg) from e
+    finally:
+        await runner.teardown()

runtime_scripts/purge_queues.py

@@ -0,0 +1,214 @@
+r"""
+Purge RabbitMQ queues and exchange.
+
+Thin CLI wrapper around the framework ``purge_queues()`` utility.
+Reads queue names from a pipeline spec or accepts them as CLI args.
+
+Usage::
+
+    # From pipeline spec
+    python -m runtime_scripts.purge_queues \
+        --config-file ./pipeline/config.yaml \
+        --pipeline-spec ./pipeline
+
+    # Explicit queue names
+    python -m runtime_scripts.purge_queues \
+        --config-file ./pipeline/config.yaml \
+        --queues jobs.parser jobs.chunker jobs.embedder
+"""
+
+import argparse
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+from basics.logging import get_logger
+from basics.logging_utils import summarize_exception_chain
+
+from runtime_scripts.lib.logging_setup import (
+    configure_logging,
+    resolve_log_level,
+)
+from runtime_scripts.lib.pipeline import (
+    DEFAULT_PIPELINE_DIR,
+    load_config,
+    load_pipeline,
+)
+from warren.exceptions import WarrenError
+from warren.pubsub.rabbitmq.aio_pika.connection import (
+    RMQConnectionManager,
+)
+from warren.pubsub.rabbitmq.aio_pika.purge import (
+    purge_queues,
+)
+from warren.pubsub.rabbitmq.config import (
+    RMQConnectionConfig,
+)
+
+
+module_logger: logging.Logger = get_logger(__name__)
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Purge RabbitMQ queues and exchange",
+    )
+    parser.add_argument(
+        "--config-file",
+        type=Path,
+        default=None,
+        help="Path to RuntimeConfig YAML. Default: ./pipeline/config.yaml",
+    )
+    parser.add_argument(
+        "--pipeline-spec",
+        type=str,
+        default=None,
+        help=(
+            "Pipeline spec location (to read queue names from worker types). "
+            f"Format: [<path>|<module>][:<var>]. Default: {DEFAULT_PIPELINE_DIR}"
+        ),
+    )
+    parser.add_argument(
+        "--queues",
+        nargs="+",
+        default=None,
+        help="Explicit queue names to purge (overrides --pipeline-spec).",
+    )
+    parser.add_argument(
+        "--exchange",
+        type=str,
+        default=None,
+        help="Exchange name to delete. Read from config if not specified.",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        default=False,
+        help="Enable DEBUG logging (default: INFO).",
+    )
+    return parser.parse_args()
+
+
+def describe_config(
+    config_file: Path | None,
+    pipeline_spec: str | None,
+    queues: list[str] | None,
+    exchange: str | None,
+    debug: bool,
+    logger: logging.Logger,
+) -> None:
+    """Log input configuration."""
+    logger.info("Configuration:")
+    logger.info(f"  config_file: {config_file}")
+    logger.info(f"  pipeline_spec: {pipeline_spec}")
+    logger.info(f"  queues: {queues}")
+    logger.info(f"  exchange: {exchange}")
+    logger.info(f"  debug: {debug}")
+
+
+async def run_purge(
+    *,
+    config_file: Path | None = None,
+    pipeline_spec: str | None = None,
+    queues: list[str] | None = None,
+    exchange: str | None = None,
+    debug: bool = False,
+    logger: logging.Logger | None = None,
+) -> None:
+    """Purge RabbitMQ queues and optionally delete the exchange.
+
+    :param config_file: path to RuntimeConfig YAML.
+    :param pipeline_spec: pipeline spec location (for deriving queue
+        names from worker types).
+    :param queues: explicit queue names (overrides pipeline_spec).
+    :param exchange: exchange name to delete. Read from config if
+        not specified.
+    :param debug: enable DEBUG logging.
+    :param logger: optional logger override.
+    """
+    log = logger or module_logger
+
+    describe_config(
+        config_file=config_file,
+        pipeline_spec=pipeline_spec,
+        queues=queues,
+        exchange=exchange,
+        debug=debug,
+        logger=log,
+    )
+
+    resolved_config_path = config_file or Path("./pipeline/config.yaml")
+
+    try:
+        config = load_config(resolved_config_path)
+    except Exception as e:
+        msg = f"Unable to load config from: {resolved_config_path}"
+        raise WarrenError(msg) from e
+
+    if queues is not None:
+        queue_names = queues
+    else:
+        spec_str = pipeline_spec or DEFAULT_PIPELINE_DIR
+        try:
+            pipeline, _ = load_pipeline(spec_str, log)
+        except Exception as e:
+            msg = f"Unable to load pipeline spec from: {spec_str}"
+            raise WarrenError(msg) from e
+
+        exchange_name = config.rabbitmq.exchange.name
+        queue_names = [f"{exchange_name}.{wt}" for wt in pipeline.workers]
+
+    exchange_to_delete = exchange or config.rabbitmq.exchange.name
+
+    log.info(f"Queues to purge: {queue_names}")
+    log.info(f"Exchange to delete: {exchange_to_delete}")
+
+    rmq_cfg = config.rabbitmq.connection
+    try:
+        connection_manager = RMQConnectionManager(
+            RMQConnectionConfig(
+                host=rmq_cfg.host,
+                port=rmq_cfg.port,
+                login=rmq_cfg.login,
+                password=rmq_cfg.password,
+            ),
+        )
+    except Exception as e:
+        msg = f"Unable to create RMQ manager for: {rmq_cfg.host}:{rmq_cfg.port}"
+        raise WarrenError(msg) from e
+
+    try:
+        await connection_manager.setup()
+    except Exception as e:
+        msg = f"Unable to connect to RabbitMQ at: {rmq_cfg.host}:{rmq_cfg.port}"
+        raise WarrenError(msg) from e
+
+    try:
+        await purge_queues(
+            connection_manager=connection_manager,
+            queue_names=queue_names,
+            exchange_name=exchange_to_delete,
+        )
+    except Exception as e:
+        msg = f"Failed to purge queues: {queue_names}, exchange: {exchange_to_delete}"
+        raise WarrenError(msg) from e
+    finally:
+        await connection_manager.teardown()
+
+
+def main() -> None:
+    global module_logger
+    args = _parse_args()
+    configure_logging(debug=args.debug)
+    module_logger = get_logger(__name__, log_level=resolve_log_level(debug=args.debug))
+
+    try:
+        asyncio.run(run_purge(**vars(args), logger=module_logger))
+    except Exception as e:
+        module_logger.error(f"Purge failed: {summarize_exception_chain(e)}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

runtime_scripts/start_job_publication_worker.py

@@ -0,0 +1,165 @@
+r"""
+Job publication worker launcher.
+
+Starts a ``JobPublicationWorkerRunner`` that consumes job messages and
+publishes their documents into the processing pipeline.
+
+Requires a ``--publisher-factory`` pointing to an async factory function
+matching ``DocumentsPublisherFactoryFunc``. The runner calls this factory
+in ``setup()`` with the shared RMQ publisher, infrastructure, config,
+and worker name.
+
+Usage::
+
+    python -m runtime_scripts.start_job_publication_worker \
+        --config-file ./pipeline/config.yaml \
+        --publisher-factory my_pipeline.publishers.factory:create_multi_type_publisher
+"""
+
+import argparse
+import asyncio
+import importlib
+import logging
+import sys
+from functools import partial
+from pathlib import Path
+
+from basics.logging import get_logger
+from basics.logging_utils import summarize_exception_chain
+
+from runtime_scripts.lib.cli import (
+    add_common_args,
+)
+from runtime_scripts.lib.logging_setup import (
+    configure_logging,
+    resolve_log_level,
+)
+from runtime_scripts.lib.runner import run
+from warren.exceptions import WarrenError
+from warren.jobs.publishing.job_publication_worker_runner import (
+    DocumentsPublisherFactoryFunc,
+    JobPublicationWorkerRunner,
+)
+
+
+module_logger: logging.Logger = get_logger(__name__)
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Start a job publication worker",
+    )
+    parser.add_argument(
+        "--publisher-factory",
+        type=str,
+        required=True,
+        help=(
+            "Dotted path to an async factory function matching "
+            "DocumentsPublisherFactoryFunc, e.g. "
+            "my.module:create_multi_type_publisher. "
+            "The factory receives (publisher, infra, config, "
+            "worker_name) and returns a JobDocumentsPublisher."
+        ),
+    )
+    add_common_args(parser)
+    return parser.parse_args()
+
+
+def describe_config(
+    publisher_factory: str,
+    config_file: str | None,
+    worker_name: str | None,
+    debug: bool,
+    logger: logging.Logger,
+) -> None:
+    """Log input configuration before any resolution or work."""
+    logger.info("Configuration:")
+    logger.info(f"  publisher_factory: {publisher_factory}")
+    logger.info(f"  config_file: {config_file}")
+    logger.info(f"  worker_name: {worker_name}")
+    logger.info(f"  debug: {debug}")
+
+
+def _load_publisher_factory(factory_path: str) -> DocumentsPublisherFactoryFunc:
+    """Import a publisher factory from ``module.path:func_name``."""
+    if ":" not in factory_path:
+        msg = f"Publisher factory must be module.path:func_name, got: {factory_path}"
+        raise ValueError(msg)
+    module_path, func_name = factory_path.rsplit(":", 1)
+    mod = importlib.import_module(module_path)
+    if not hasattr(mod, func_name):
+        msg = f"Module {module_path} has no attribute '{func_name}'"
+        raise AttributeError(msg)
+    return getattr(mod, func_name)
+
+
+async def start_job_publication_worker(
+    *,
+    publisher_factory: str,
+    config_file: str | None = None,
+    worker_name: str | None = None,
+    debug: bool = False,
+    logger: logging.Logger | None = None,
+) -> None:
+    """Start a job publication worker.
+
+    :param publisher_factory: dotted path to the publisher factory
+        function (``module:func``) matching
+        ``DocumentsPublisherFactoryFunc``.
+    :param config_file: path to RuntimeConfig YAML.
+    :param worker_name: unique worker instance name.
+    :param debug: enable DEBUG logging.
+    :param logger: optional logger override.
+    """
+    log = logger or module_logger
+
+    describe_config(
+        publisher_factory=publisher_factory,
+        config_file=config_file,
+        worker_name=worker_name,
+        debug=debug,
+        logger=log,
+    )
+
+    try:
+        pub_factory_func = _load_publisher_factory(publisher_factory)
+    except Exception as e:
+        msg = f"Unable to load publisher factory: {publisher_factory}"
+        raise WarrenError(msg) from e
+
+    resolved_config = (
+        Path(config_file) if config_file else Path("./pipeline/config.yaml")
+    )
+
+    runner_factory = partial(
+        JobPublicationWorkerRunner,
+        documents_publisher_factory=pub_factory_func,
+    )
+
+    await run(
+        runner_factory_func=runner_factory,
+        config_file=resolved_config,
+        worker_name=worker_name,
+        worker_name_prefix="publication-worker",
+        debug=debug,
+        logger=log,
+    )
+
+
+def main() -> None:
+    global module_logger
+    args = _parse_args()
+    configure_logging(debug=args.debug)
+    module_logger = get_logger(__name__, log_level=resolve_log_level(debug=args.debug))
+
+    try:
+        asyncio.run(start_job_publication_worker(**vars(args), logger=module_logger))
+    except Exception as e:
+        module_logger.error(
+            f"Start job publication worker failed: {summarize_exception_chain(e)}"
+        )
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()