moxtral 0.4.0__py3-none-any.whl

moxtral/__init__.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("moxtral")
+except PackageNotFoundError:  # e.g. vendored source tree without installation
+    __version__ = "0.0.0+unknown"
+
+__all__ = ["__version__"]

moxtral/__main__.py

@@ -0,0 +1,4 @@
+from moxtral.cli.main import cli
+
+if __name__ == "__main__":
+    cli()

moxtral/cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line interface for moxtral."""

moxtral/cli/agent.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from importlib import resources
+from typing import TYPE_CHECKING
+
+import click
+
+from moxtral.formatters import serialize_json
+from moxtral.schema import record_schema
+
+if TYPE_CHECKING:
+    from moxtral.cli.main import AppContext
+
+
+@click.command()
+@click.option(
+    "--schema",
+    "show_schema",
+    is_flag=True,
+    help="Print the JSON Schema describing --json output records.",
+)
+@click.pass_obj
+def agent(context: AppContext, show_schema: bool) -> None:
+    """Print agent-oriented usage documentation for this CLI."""
+    if show_schema:
+        context.consoles.write_stdout(serialize_json(record_schema()))
+        return
+    guide = (
+        resources.files("moxtral")
+        .joinpath("data/agent_guide.md")
+        .read_text(encoding="utf-8")
+    )
+    context.consoles.write_stdout(guide)

moxtral/cli/common.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import math
+import os
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Protocol, cast
+
+import click
+
+from moxtral.config import ConfigStore
+from moxtral.console import sanitize_terminal_text
+from moxtral.errors import (
+    ConfigError,
+    format_debug_exception,
+    redact,
+    translate_exception,
+)
+from moxtral.models import ApiResult, InputSource, JSONMapping, JSONValue
+
+
+def positive_days(
+    context: click.Context,
+    parameter: click.Parameter,
+    value: float,
+) -> float:
+    """Validate a look-back window expressed as a positive, finite number of days."""
+    if not math.isfinite(value) or value <= 0:
+        raise click.BadParameter(
+            "must be a positive number of days",
+            ctx=context,
+            param=parameter,
+        )
+    return value
+
+
+def nonnegative_integer(
+    context: click.Context,
+    parameter: click.Parameter,
+    value: int | None,
+) -> int | None:
+    """Validate an optional nonnegative integer click option."""
+    if value is not None and value < 0:
+        raise click.BadParameter(
+            "must be a nonnegative integer",
+            ctx=context,
+            param=parameter,
+        )
+    return value
+
+
+class CommandConsoles(Protocol):
+    def write_stdout(self, payload: str) -> None: ...
+
+    def write_stderr(self, payload: str) -> None: ...
+
+
+class CommandContext(Protocol):
+    @property
+    def config_path(self) -> Path: ...
+
+    @property
+    def debug(self) -> bool: ...
+
+    @property
+    def consoles(self) -> CommandConsoles: ...
+
+
+def safe_terminal_text(text: str, secrets: tuple[str, ...]) -> str:
+    """Sanitize terminal controls and redact every resulting secret variant."""
+    safe_text = sanitize_terminal_text(text)
+    secret_variants: list[str] = []
+    for secret in secrets:
+        if not secret.strip():
+            continue
+        secret_variants.append(secret)
+        secret_variants.append(sanitize_terminal_text(secret))
+    return redact(safe_text, secret_variants)
+
+
+def candidate_secrets(context: CommandContext) -> tuple[str, ...]:
+    """Collect potential keys without allowing config errors to mask input errors."""
+    secrets: list[str] = []
+    environment_key = os.environ.get("MISTRAL_API_KEY")
+    if environment_key is not None and environment_key.strip():
+        secrets.append(environment_key)
+
+    try:
+        configured_key = ConfigStore(context.config_path, environ={}).load().api_key
+    except ConfigError:
+        configured_key = None
+    if configured_key is not None:
+        secrets.append(configured_key)
+    return tuple(dict.fromkeys(secrets))
+
+
+def extend_secrets(secrets: tuple[str, ...], secret: str) -> tuple[str, ...]:
+    """Append a newly resolved secret while preserving order and uniqueness."""
+    if secret in secrets:
+        return secrets
+    return (*secrets, secret)
+
+
+def _redact_json_value(
+    value: JSONValue,
+    secrets: tuple[str, ...],
+) -> JSONValue:
+    if isinstance(value, str):
+        return redact(value, secrets)
+    if isinstance(value, list):
+        return [_redact_json_value(item, secrets) for item in value]
+    if isinstance(value, Mapping):
+        return _redact_json_mapping(cast(JSONMapping, value), secrets)
+    return value
+
+
+def _redact_json_mapping(
+    mapping: JSONMapping,
+    secrets: tuple[str, ...],
+) -> dict[str, JSONValue]:
+    redacted_mapping: dict[str, JSONValue] = {}
+    for key, value in mapping.items():
+        safe_key = redact(key, secrets)
+        unique_key = safe_key
+        suffix = 1
+        while unique_key in redacted_mapping:
+            unique_key = f"{safe_key}#{suffix}"
+            suffix += 1
+        redacted_mapping[unique_key] = _redact_json_value(value, secrets)
+    return redacted_mapping
+
+
+def _redact_source(
+    source: InputSource,
+    secrets: tuple[str, ...],
+) -> InputSource:
+    safe_path = None if source.path is None else Path(redact(str(source.path), secrets))
+    return InputSource(
+        kind=source.kind,
+        value=redact(source.value, secrets),
+        filename=redact(source.filename, secrets),
+        path=safe_path,
+        ocr_kind=source.ocr_kind,
+    )
+
+
+def redact_result(
+    result: ApiResult,
+    secrets: tuple[str, ...],
+) -> ApiResult:
+    """Return an API result with secrets recursively removed."""
+    return ApiResult(
+        operation=result.operation,
+        source=_redact_source(result.source, secrets),
+        request_metadata=_redact_json_mapping(result.request_metadata, secrets),
+        response=_redact_json_mapping(result.response, secrets),
+        created_at=result.created_at,
+    )
+
+
+def report_error(
+    context: CommandContext,
+    error: Exception,
+    *,
+    secrets: tuple[str, ...],
+    setup_debug_context: str,
+    source_debug_prefix: str,
+    source: str | None = None,
+) -> None:
+    """Translate and safely report a setup or per-source command error."""
+    translated = translate_exception(error)
+    line = (
+        f"Setup error: {translated}\n"
+        if source is None
+        else f"{source}: {translated}\n"
+    )
+    debug_context = (
+        setup_debug_context if source is None else f"{source_debug_prefix}: {source}"
+    )
+    context.consoles.write_stderr(safe_terminal_text(line, secrets))
+    if context.debug:
+        write_debug_exception(
+            context,
+            error,
+            secrets=secrets,
+            debug_context=debug_context,
+        )
+
+
+def write_debug_exception(
+    context: CommandContext,
+    error: Exception,
+    *,
+    secrets: tuple[str, ...],
+    debug_context: str,
+) -> None:
+    """Write traceback diagnostics through the shared safe terminal boundary."""
+    formatted = format_debug_exception(
+        error,
+        context=debug_context,
+    )
+    context.consoles.write_stderr(safe_terminal_text(formatted, secrets))

moxtral/cli/config.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING, NoReturn, cast
+
+import click
+import tomli_w
+
+from moxtral.cli.common import (
+    candidate_secrets,
+    extend_secrets,
+    safe_terminal_text,
+    write_debug_exception,
+)
+from moxtral.config import ConfigStore
+from moxtral.errors import ConfigError
+
+if TYPE_CHECKING:
+    from moxtral.cli.main import AppContext
+
+
+@click.group()
+def config() -> None:
+    """Manage CLI configuration."""
+
+
+def _raise_config_error(
+    context: AppContext,
+    error: ConfigError,
+    *,
+    secrets: tuple[str, ...],
+    debug_context: str,
+) -> NoReturn:
+    if context.debug:
+        write_debug_exception(
+            context,
+            error,
+            secrets=secrets,
+            debug_context=debug_context,
+        )
+    raise click.ClickException(safe_terminal_text(str(error), secrets)) from error
+
+
+@config.command(
+    "set",
+    context_settings={
+        "allow_extra_args": True,
+        "ignore_unknown_options": True,
+    },
+)
+@click.argument("name", type=click.Choice(["api-key"]))
+@click.option(
+    "--stdin",
+    "read_stdin",
+    is_flag=True,
+    help="Read the value from standard input.",
+)
+@click.pass_context
+def set_value(context: click.Context, name: str, read_stdin: bool) -> None:
+    """Set a configuration value."""
+    if context.args:
+        raise click.UsageError(
+            "API keys must be provided through the prompt or --stdin.",
+            ctx=context,
+        )
+
+    if read_stdin:
+        try:
+            value = sys.stdin.read().rstrip("\r\n")
+        except UnicodeError as error:
+            raise click.ClickException(
+                "Could not read API key from standard input."
+            ) from error
+        if not value:
+            raise click.ClickException("Standard input must not be empty.")
+        if "\r" in value or "\n" in value:
+            raise click.ClickException("Standard input must contain a single line.")
+    else:
+        value = click.prompt(
+            "API key",
+            hide_input=True,
+            confirmation_prompt=True,
+        )
+
+    app_context = cast("AppContext", context.obj)
+    try:
+        ConfigStore(app_context.config_path).set(name, value)
+    except ConfigError as error:
+        secrets = extend_secrets(candidate_secrets(app_context), value)
+        _raise_config_error(
+            app_context,
+            error,
+            secrets=secrets,
+            debug_context=f"setting configuration {name}",
+        )
+
+    click.echo("Configuration updated.")
+
+
+@config.command("show")
+@click.pass_obj
+def show(context: AppContext) -> None:
+    """Show configuration with secrets redacted."""
+    try:
+        values = ConfigStore(context.config_path).redacted()
+    except ConfigError as error:
+        _raise_config_error(
+            context,
+            error,
+            secrets=candidate_secrets(context),
+            debug_context="showing configuration",
+        )
+
+    click.echo(tomli_w.dumps(values), nl=False)
+
+
+@config.command("unset")
+@click.argument("name", type=click.Choice(["api-key"]))
+@click.pass_obj
+def unset(context: AppContext, name: str) -> None:
+    """Remove a configuration value."""
+    try:
+        removed = ConfigStore(context.config_path).unset(name)
+    except ConfigError as error:
+        _raise_config_error(
+            context,
+            error,
+            secrets=candidate_secrets(context),
+            debug_context=f"unsetting configuration {name}",
+        )
+
+    if removed:
+        click.echo(f"{name} removed.")
+    else:
+        click.echo(f"{name} already absent.")
+
+
+@config.command("path")
+@click.pass_obj
+def show_path(context: AppContext) -> None:
+    """Print the effective configuration path."""
+    click.echo(context.config_path)

moxtral/cli/main.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import click
+
+from moxtral import __version__
+from moxtral.cli.agent import agent
+from moxtral.cli.config import config
+from moxtral.cli.ocr import ocr
+from moxtral.cli.transcribe import transcribe
+from moxtral.console import ConsoleBundle, create_console_bundle
+
+DEFAULT_CONFIG_PATH = Path("~/.moxtral/config.toml").expanduser()
+
+
+@dataclass(frozen=True, slots=True)
+class AppContext:
+    config_path: Path
+    debug: bool
+    consoles: ConsoleBundle
+
+
+@click.group(
+    context_settings={"help_option_names": ["-h", "--help"]},
+    epilog=(
+        "Run 'moxtral agent' for agent-oriented usage docs and "
+        "'moxtral agent --schema' for the JSON output schema."
+    ),
+)
+@click.version_option(version=__version__)
+@click.option("--debug", is_flag=True, help="Show detailed error information.")
+@click.option(
+    "--config",
+    "config_path",
+    type=click.Path(path_type=Path, dir_okay=False),
+    default=DEFAULT_CONFIG_PATH,
+    show_default=True,
+    help="Path to the configuration file.",
+)
+@click.pass_context
+def cli(ctx: click.Context, debug: bool, config_path: Path) -> None:
+    """Work with Mistral OCR and audio transcription."""
+    ctx.obj = AppContext(
+        config_path=config_path,
+        debug=debug,
+        consoles=create_console_bundle(),
+    )
+
+
+cli.add_command(agent)
+cli.add_command(config)
+cli.add_command(ocr)
+cli.add_command(transcribe)

moxtral/cli/ocr.py

@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, cast
+
+import click
+
+from moxtral.cli.common import nonnegative_integer, positive_days
+from moxtral.cli.runner import BatchPlan, DedupeOptions, OutputOptions, run_batch
+from moxtral.formatters import format_ocr_markdown
+from moxtral.mistral_client import MistralGateway
+from moxtral.models import (
+    DEFAULT_RETRIES,
+    Confidence,
+    OcrRequest,
+    Operation,
+    OutputFormat,
+    TableFormat,
+    build_ocr_request,
+    ocr_request_metadata,
+)
+from moxtral.services.ocr import OcrGateway, OcrService
+from moxtral.sources import resolve_source
+from moxtral.storage import ResultStore
+
+if TYPE_CHECKING:
+    from moxtral.cli.main import AppContext
+
+
+def create_gateway(api_key: str) -> OcrGateway:
+    """Create the production OCR gateway."""
+    return MistralGateway(api_key)
+
+
+def create_result_store() -> ResultStore:
+    """Create the production result store."""
+    return ResultStore()
+
+
+@click.command()
+@click.argument("sources", metavar="SOURCE...", nargs=-1, required=True)
+@click.option(
+    "--model",
+    default="mistral-ocr-latest",
+    show_default=True,
+    help="OCR model.",
+)
+@click.option(
+    "--pages",
+    help="Page numbers or ranges in API syntax, such as 0,2-4.",
+)
+@click.option(
+    "--table-format",
+    type=click.Choice(["inline", "markdown", "html"]),
+    default="inline",
+    show_default=True,
+    help="How tables are represented in the OCR response.",
+)
+@click.option(
+    "--extract-header",
+    is_flag=True,
+    help="Extract page headers.",
+)
+@click.option(
+    "--extract-footer",
+    is_flag=True,
+    help="Extract page footers.",
+)
+@click.option(
+    "--include-images",
+    is_flag=True,
+    help="Include extracted image data in the response.",
+)
+@click.option(
+    "--image-limit",
+    type=int,
+    callback=nonnegative_integer,
+    help="Maximum number of images to extract (requires --include-images).",
+)
+@click.option(
+    "--image-min-size",
+    type=int,
+    callback=nonnegative_integer,
+    help="Minimum extracted image size (requires --include-images).",
+)
+@click.option(
+    "--include-blocks",
+    is_flag=True,
+    help="Include structured OCR blocks.",
+)
+@click.option(
+    "--confidence",
+    type=click.Choice(["none", "page", "word"]),
+    default="none",
+    show_default=True,
+    help="Confidence score granularity.",
+)
+@click.option(
+    "--output-dir",
+    type=click.Path(path_type=Path, file_okay=False),
+    help="Directory in which to save result files.",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["md", "json", "both"]),
+    default="both",
+    show_default=True,
+    help="Result file format to save.",
+)
+@click.option(
+    "--timeout",
+    type=float,
+    default=300.0,
+    show_default=True,
+    help="Request timeout in seconds.",
+)
+@click.option(
+    "--retries",
+    type=int,
+    default=DEFAULT_RETRIES,
+    show_default=True,
+    callback=nonnegative_integer,
+    help=(
+        "Retry attempts for rate-limited, server-error, and connection "
+        "failures (0 disables)."
+    ),
+)
+@click.option(
+    "--stdout",
+    "write_stdout",
+    is_flag=True,
+    help="Also write OCR Markdown to standard output.",
+)
+@click.option(
+    "--json",
+    "write_json",
+    is_flag=True,
+    help="Write NDJSON result records to standard output (one per source).",
+)
+@click.option(
+    "--quiet",
+    is_flag=True,
+    help="Suppress progress and summary output.",
+)
+@click.option(
+    "--no-save",
+    is_flag=True,
+    help="Do not save result files (requires --json or --stdout).",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Validate sources and options without calling the API.",
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    help="Process the source even if an identical recent result exists.",
+)
+@click.option(
+    "--dedupe-window",
+    type=float,
+    default=30.0,
+    show_default=True,
+    metavar="DAYS",
+    callback=positive_days,
+    help="Look-back window in days for skipping identical, already-saved results.",
+)
+@click.pass_obj
+def ocr(
+    context: AppContext,
+    sources: tuple[str, ...],
+    model: str,
+    pages: str | None,
+    table_format: str,
+    extract_header: bool,
+    extract_footer: bool,
+    include_images: bool,
+    image_limit: int | None,
+    image_min_size: int | None,
+    include_blocks: bool,
+    confidence: str,
+    output_dir: Path | None,
+    output_format: str,
+    timeout: float,
+    retries: int,
+    write_stdout: bool,
+    write_json: bool,
+    quiet: bool,
+    no_save: bool,
+    dry_run: bool,
+    force: bool,
+    dedupe_window: float,
+) -> None:
+    """Extract readable text from local documents, images, or HTTP(S) URLs."""
+    selected_table_format = (
+        None if table_format == "inline" else cast(TableFormat, table_format)
+    )
+    selected_confidence = None if confidence == "none" else cast(Confidence, confidence)
+
+    def build_request(source_value: str) -> OcrRequest:
+        return build_ocr_request(
+            source=resolve_source(source_value, Operation.OCR),
+            model=model,
+            pages=pages,
+            table_format=selected_table_format,
+            extract_header=extract_header,
+            extract_footer=extract_footer,
+            include_images=include_images,
+            image_limit=image_limit,
+            image_min_size=image_min_size,
+            include_blocks=include_blocks,
+            confidence=selected_confidence,
+            timeout_seconds=timeout,
+            retries=retries,
+        )
+
+    run_batch(
+        context,
+        sources,
+        BatchPlan(
+            setup_debug_context="setting up OCR command",
+            source_debug_prefix="OCR source",
+            operation=Operation.OCR,
+            build_request=build_request,
+            request_metadata=ocr_request_metadata,
+            create_service=lambda api_key: OcrService(create_gateway(api_key)),
+            create_store=lambda: create_result_store(),
+            format_markdown=format_ocr_markdown,
+        ),
+        OutputOptions(
+            output_format=OutputFormat(output_format),
+            output_dir=output_dir,
+            write_markdown_stdout=write_stdout,
+            write_json_stdout=write_json,
+            quiet=quiet,
+            no_save=no_save,
+            dry_run=dry_run,
+        ),
+        DedupeOptions(force=force, window_days=dedupe_window),
+    )