sci-etl-cli 0.1.0__py3-none-any.whl

sci_etl_cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line tool for running sci-etl-core extraction pipelines from a YAML file."""

sci_etl_cli/__main__.py

@@ -0,0 +1,3 @@
+from sci_etl_cli.app import main
+
+main()

sci_etl_cli/app.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from importlib.metadata import version
+from typing import Any
+
+import click
+from sci_etl_core.exceptions import SciEtlError
+
+from sci_etl_cli.commands.init import init_command
+from sci_etl_cli.commands.parse import parse_command
+from sci_etl_cli.commands.run import run_command
+from sci_etl_cli.commands.search import search_command
+from sci_etl_cli.commands.status import status_command
+from sci_etl_cli.commands.validate import validate_command
+from sci_etl_cli.errors import to_cli_failure
+
+_EXIT_CODES = """\b
+Exit codes:
+  0    success
+  1    run aborted, or a request or parse failed
+  2    invalid command-line usage
+  3    configuration problem
+  4    missing Python package
+  130  interrupted after saving state"""
+
+
+class SciEtlGroup(click.Group):
+    """Command group that reports library errors as one line and an exit code."""
+
+    def invoke(self, ctx: click.Context) -> Any:
+        try:
+            return super().invoke(ctx)
+        except (SciEtlError, ModuleNotFoundError) as error:
+            raise to_cli_failure(error) from error
+
+
+def _print_version(ctx: click.Context, _param: click.Parameter, value: bool) -> None:
+    if not value or ctx.resilient_parsing:
+        return
+    click.echo(f"sci-etl {version('sci-etl-cli')} (sci-etl-core {version('sci-etl-core')})")
+    ctx.exit()
+
+
+@click.group(
+    cls=SciEtlGroup,
+    context_settings={"help_option_names": ["-h", "--help"]},
+    epilog=_EXIT_CODES,
+)
+@click.option(
+    "--version",
+    is_flag=True,
+    expose_value=False,
+    is_eager=True,
+    callback=_print_version,
+    help="Show the CLI and library versions and exit.",
+)
+def cli() -> None:
+    """Run sci-etl-core extraction pipelines from a YAML config file."""
+
+
+cli.add_command(init_command)
+cli.add_command(validate_command)
+cli.add_command(search_command)
+cli.add_command(run_command)
+cli.add_command(status_command)
+cli.add_command(parse_command)
+
+
+def main() -> None:
+    cli(prog_name="sci-etl")

sci_etl_cli/assembly.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Callable
+
+from sci_etl_core.exceptions import ConfigurationError
+
+if TYPE_CHECKING:
+    import logging
+
+    import httpx
+    from sci_etl_core import (
+        AsyncETLPipeline,
+        AsyncExporter,
+        AsyncExtractor,
+        AsyncLLMClient,
+        AsyncOpenAICompatibleClient,
+        AsyncStateManager,
+        PipelineMetadata,
+    )
+
+    from sci_etl_cli.settings import CliConfig
+
+
+def read_prompt(path: Path) -> str:
+    """Return a prompt file's text.
+
+    Raises:
+        ConfigurationError: The file is missing, unreadable, not UTF-8, or blank.
+    """
+    try:
+        text = path.read_text(encoding="utf-8")
+    except FileNotFoundError as exc:
+        raise ConfigurationError(f"Prompt file not found: {path}") from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise ConfigurationError(f"Prompt file could not be read: {path}: {exc}") from exc
+    if not text.strip():
+        raise ConfigurationError(f"Prompt file is empty: {path}")
+    return text
+
+
+def build_http_client(config: CliConfig) -> httpx.AsyncClient:
+    from sci_etl_core.http_async import build_async_client
+
+    return build_async_client(timeout=config.http.timeout, user_agent=config.http.user_agent)
+
+
+def build_llm_client(config: CliConfig) -> AsyncOpenAICompatibleClient:
+    from sci_etl_core import AsyncOpenAICompatibleClient
+
+    return AsyncOpenAICompatibleClient(
+        api_key=config.llm.api_key,
+        base_url=config.llm.base_url,
+        model=config.llm.model,
+        default_timeout=config.llm.timeout,
+    )
+
+
+def build_extractor(
+    config: CliConfig, http_client: httpx.AsyncClient, log: Callable[[str], None]
+) -> AsyncExtractor:
+    from sci_etl_core import AsyncArxivExtractor
+    from sci_etl_core.parsers import LatexTarballParser, PdfPlumberParser
+
+    return AsyncArxivExtractor(
+        client=http_client,
+        pdf_parser=PdfPlumberParser(),
+        latex_parser=LatexTarballParser(),
+        max_retries=config.http.max_retries,
+        backoff_factor=config.http.backoff_factor,
+        sleep_before_search=config.pipeline.search_delay,
+        logger=log,
+    )
+
+
+def build_state_manager(config: CliConfig) -> AsyncStateManager:
+    if config.state.backend == "sqlite":
+        from sci_etl_core import AsyncSqliteStateManager
+
+        return AsyncSqliteStateManager(config.state.database)
+    from sci_etl_core import AsyncFileStateManager
+
+    return AsyncFileStateManager(config.state.processed_ids, config.state.metadata)
+
+
+def build_exporter(config: CliConfig) -> AsyncExporter:
+    from sci_etl_core import AsyncCsvUpsertExporter
+    from sci_etl_core.processors import DefaultKeyNormalizer
+
+    export = config.export
+    return AsyncCsvUpsertExporter(
+        key_column=export.key_column,
+        value_columns=list(export.value_columns),
+        normalizer=DefaultKeyNormalizer(),
+        numeric_clip=dict(export.numeric_clip),
+        escape_formulas=export.escape_formulas,
+    )
+
+
+def build_pipeline(
+    config: CliConfig,
+    log: logging.Logger,
+    http_client: httpx.AsyncClient,
+    llm_client: AsyncLLMClient,
+    state_manager: AsyncStateManager,
+    relevance_prompt: str,
+    extraction_prompt: str,
+) -> AsyncETLPipeline:
+    from sci_etl_core import AsyncETLPipeline, AsyncLLMEntityExtractor, AsyncLLMRelevanceFilter
+
+    from sci_etl_cli.reporting import ListingReporter
+
+    return AsyncETLPipeline(
+        extractor=ListingReporter(build_extractor(config, http_client, log.info), log.info),
+        relevance_filter=AsyncLLMRelevanceFilter(llm_client=llm_client, system_prompt=relevance_prompt),
+        entity_extractor=AsyncLLMEntityExtractor(
+            llm_client=llm_client,
+            system_prompt=extraction_prompt,
+            result_key=config.prompts.result_key,
+            timeout=config.llm.timeout,
+        ),
+        exporter=build_exporter(config),
+        state_manager=state_manager,
+        destination=str(config.export.destination),
+        max_concurrency=config.pipeline.max_workers,
+        logger=log.warning,
+        closeables=[http_client, llm_client, state_manager],
+    )
+
+
+async def read_state(config: CliConfig) -> tuple[set[str], PipelineMetadata]:
+    """Load processed ids and metadata without creating state that does not exist yet."""
+    from sci_etl_core.models import PipelineMetadata
+
+    if config.state.backend == "sqlite" and not config.state.database.exists():
+        return set(), PipelineMetadata()
+    manager = build_state_manager(config)
+    try:
+        return await manager.load_processed_ids(), await manager.load_metadata()
+    finally:
+        aclose = getattr(manager, "aclose", None)
+        if aclose is not None:
+            await aclose()

sci_etl_cli/commands/init.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from importlib import resources
+from pathlib import Path
+
+import click
+from rich.markup import escape
+
+from sci_etl_cli.output import stdout_console
+
+_TEMPLATES: tuple[tuple[str, str], ...] = (
+    ("config.yaml", "config.yaml"),
+    ("relevance.txt", "prompts/relevance.txt"),
+    ("extraction.txt", "prompts/extraction.txt"),
+    ("env.example", ".env.example"),
+)
+
+
+@click.command("init", short_help="Create a starter project.")
+@click.argument("directory", required=False, default=".", type=click.Path(file_okay=False, path_type=Path))
+@click.option("--force", is_flag=True, help="Overwrite files that already exist.")
+def init_command(directory: Path, force: bool) -> None:
+    """Create a starter project in DIRECTORY (default: the current folder).
+
+    Writes config.yaml, prompts/relevance.txt, prompts/extraction.txt and
+    .env.example. The example collects hot Jupiter measurements from arXiv;
+    change the query, prompts and export columns for your own subject.
+    """
+    targets = [(template, directory / relative) for template, relative in _TEMPLATES]
+    existing = [path for _template, path in targets if path.exists()]
+    if existing and not force:
+        names = ", ".join(str(path) for path in existing)
+        raise click.UsageError(f"Refusing to overwrite {names}; pass --force to replace them.")
+
+    templates = resources.files("sci_etl_cli").joinpath("templates")
+    console = stdout_console()
+    for template, path in targets:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(templates.joinpath(template).read_text(encoding="utf-8"), encoding="utf-8")
+        console.print(f"[green]wrote[/] {escape(str(path))}", soft_wrap=True)
+    console.print(
+        "\nNext: copy .env.example to .env, set LLM_API_KEY, then run "
+        f"[bold]sci-etl validate {escape(str(directory / 'config.yaml'))}[/]",
+        soft_wrap=True,
+    )

sci_etl_cli/commands/parse.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import click
+from sci_etl_core.parsers.reference_trimmer import trim_after_references
+
+from sci_etl_cli.errors import CliFailure, ExitCode
+
+if TYPE_CHECKING:
+    from sci_etl_core.parsers.base import Parser
+
+_FORMATS = ("auto", "pdf", "latex", "html")
+_PDF_MAGIC = b"%PDF"
+_GZIP_MAGIC = b"\x1f\x8b"
+_TAR_MAGIC = b"ustar"
+_TAR_MAGIC_OFFSET = 257
+
+
+@click.command("parse", short_help="Print the text extracted from a PDF, e-print or HTML file.")
+@click.argument("file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option(
+    "--format",
+    "file_format",
+    type=click.Choice(_FORMATS),
+    default="auto",
+    show_default=True,
+    help="How to read FILE. auto recognizes PDFs, arXiv e-prints and HTML.",
+)
+@click.option("--trim-references", is_flag=True, help="Cut the text at the references or acknowledgments.")
+def parse_command(file: Path, file_format: str, trim_references: bool) -> None:
+    """Print the text a bundled parser extracts from FILE.
+
+    Accepts a PDF, an arXiv e-print (a tarball or gzipped TeX file) or an HTML
+    page. Use it to check full-text quality without running a pipeline.
+    """
+    content = file.read_bytes()
+    parser = build_parser(detect_format(content) if file_format == "auto" else file_format)
+    text = parser.extract_text(content)
+    if trim_references:
+        text = trim_after_references(text) or text
+    if not text.strip():
+        raise CliFailure(f"No text could be extracted from {file}", ExitCode.FAILURE)
+    click.echo(text)
+
+
+def detect_format(content: bytes) -> str:
+    stripped = content.lstrip()
+    if stripped.startswith(_PDF_MAGIC):
+        return "pdf"
+    tar_magic = content[_TAR_MAGIC_OFFSET : _TAR_MAGIC_OFFSET + len(_TAR_MAGIC)]
+    if content.startswith(_GZIP_MAGIC) or tar_magic == _TAR_MAGIC:
+        return "latex"
+    if stripped.startswith(b"<"):
+        return "html"
+    raise click.UsageError("Could not tell the file's format; pass --format pdf, latex or html.")
+
+
+def build_parser(file_format: str) -> Parser:
+    if file_format == "pdf":
+        from sci_etl_core.parsers.pdf import PdfPlumberParser
+
+        return PdfPlumberParser()
+    if file_format == "latex":
+        from sci_etl_core.parsers.latex import LatexTarballParser
+
+        return LatexTarballParser()
+    from sci_etl_core.parsers.html import HtmlTextParser
+
+    return HtmlTextParser()

sci_etl_cli/commands/run.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+import click
+from sci_etl_core.exceptions import PipelineAborted
+from sci_etl_core.signals import ShutdownSignal
+
+from sci_etl_cli import assembly, shutdown
+from sci_etl_cli.errors import ExitCode, describe_abort
+from sci_etl_cli.options import config_argument, require_api_key, require_query
+from sci_etl_cli.output import run_logger, stderr_console
+from sci_etl_cli.settings import CliConfig, load_cli_config
+
+
+@click.command("run")
+@config_argument
+@click.option(
+    "--limit",
+    type=click.IntRange(min=0),
+    help="Stop after this many relevant records. Overrides pipeline.max_records.",
+)
+@click.option(
+    "--page-size",
+    type=click.IntRange(min=1),
+    help="Listing entries per request. Overrides pipeline.page_size.",
+)
+@click.option(
+    "--workers",
+    type=click.IntRange(min=1),
+    help="Records processed at once. Overrides pipeline.max_workers.",
+)
+@click.option(
+    "--rescan",
+    is_flag=True,
+    help="Start at offset 0 to pick up new submissions; processed records are skipped.",
+)
+@click.option(
+    "--start-index",
+    type=click.IntRange(min=0),
+    help="Start at this listing offset instead of the saved one.",
+)
+@click.option(
+    "--log-file",
+    type=click.Path(dir_okay=False, path_type=Path),
+    help="Write the log to this file. Overrides logging.file.",
+)
+def run_command(
+    config_path: Path,
+    limit: int | None,
+    page_size: int | None,
+    workers: int | None,
+    rescan: bool,
+    start_index: int | None,
+    log_file: Path | None,
+) -> None:
+    """Run the pipeline for CONFIG, resuming from its saved state.
+
+    Searches arXiv, asks the LLM which papers are relevant, extracts entities
+    from their full text and upserts them into the export CSV. Press Ctrl+C
+    once to stop after saving state; the next run retries unfinished records.
+    """
+    if rescan and start_index is not None:
+        raise click.UsageError("Use either --rescan or --start-index, not both.")
+    config = apply_overrides(
+        load_cli_config(config_path), limit=limit, page_size=page_size, workers=workers, log_file=log_file
+    )
+    require_query(config.pipeline.search_query)
+    require_api_key(config)
+    relevance_prompt = assembly.read_prompt(config.prompts.relevance)
+    extraction_prompt = assembly.read_prompt(config.prompts.extraction)
+
+    ctx = click.get_current_context()
+    with run_logger(config.logging.level, config.logging.file, stderr_console()) as log:
+        try:
+            processed = asyncio.run(
+                execute(config, log, relevance_prompt, extraction_prompt, 0 if rescan else start_index)
+            )
+        except PipelineAborted as aborted:
+            log.error(f"Run aborted: {describe_abort(aborted)}")
+            ctx.exit(ExitCode.FAILURE)
+        if processed is None:
+            log.warning("Interrupted; state was saved. Run the same command again to continue.")
+            ctx.exit(ExitCode.INTERRUPTED)
+        log.info(f"Processed {describe_count(processed)} into {config.export.destination}")
+
+
+def apply_overrides(
+    config: CliConfig,
+    *,
+    limit: int | None,
+    page_size: int | None,
+    workers: int | None,
+    log_file: Path | None,
+) -> CliConfig:
+    requested = {"max_records": limit, "page_size": page_size, "max_workers": workers}
+    pipeline = config.pipeline.model_copy(update={name: value for name, value in requested.items() if value is not None})
+    logging_config = config.logging if log_file is None else config.logging.model_copy(update={"file": log_file})
+    return config.model_copy(update={"pipeline": pipeline, "logging": logging_config})
+
+
+def describe_count(count: int) -> str:
+    return f"{count} relevant record{'' if count == 1 else 's'}"
+
+
+async def execute(
+    config: CliConfig,
+    log: logging.Logger,
+    relevance_prompt: str,
+    extraction_prompt: str,
+    start_index: int | None,
+) -> int | None:
+    http_client = assembly.build_http_client(config)
+    llm_client = assembly.build_llm_client(config)
+    state_manager = assembly.build_state_manager(config)
+    pipeline = assembly.build_pipeline(
+        config, log, http_client, llm_client, state_manager, relevance_prompt, extraction_prompt
+    )
+    log.info(
+        f"Starting run for {config.pipeline.search_query!r}, up to {describe_count(config.pipeline.max_records)}"
+    )
+    async with pipeline:
+        return await shutdown.run_until_interrupted(
+            pipeline.run(
+                query=config.pipeline.search_query,
+                page_size=config.pipeline.page_size,
+                total_limit=config.pipeline.max_records,
+                sleep_between=config.pipeline.sleep_between,
+                start_index=start_index,
+            ),
+            ShutdownSignal(logger=log.warning),
+            state_manager,
+        )

sci_etl_cli/commands/search.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+import click
+from rich.markup import escape
+from rich.table import Table
+
+from sci_etl_cli import assembly
+from sci_etl_cli.options import config_argument, require_query
+from sci_etl_cli.output import discard, print_json, stdout_console
+from sci_etl_cli.settings import CliConfig, load_cli_config
+
+_ARXIV_MAX_RESULTS = 2000
+
+
+@dataclass(frozen=True, slots=True)
+class ListedRecord:
+    record_id: str
+    title: str
+    url: str | None
+    processed: bool
+
+
+@click.command("search", short_help="Preview arXiv results without calling the LLM.")
+@config_argument
+@click.option("--query", help="Search this instead of pipeline.search_query.")
+@click.option(
+    "--limit",
+    type=click.IntRange(1, _ARXIV_MAX_RESULTS),
+    default=10,
+    show_default=True,
+    help="Number of entries to request.",
+)
+@click.option(
+    "--start-index",
+    type=click.IntRange(min=0),
+    default=0,
+    show_default=True,
+    help="Listing offset to start from.",
+)
+@click.option("--json", "as_json", is_flag=True, help="Print JSON instead of a table.")
+def search_command(config_path: Path, query: str | None, limit: int, start_index: int, as_json: bool) -> None:
+    """Show one page of arXiv results for CONFIG without calling the LLM.
+
+    Each entry is marked new or processed according to the saved state, so a
+    query can be tuned before spending tokens. Nothing is written.
+    """
+    config = load_cli_config(config_path)
+    search_query = require_query(query or config.pipeline.search_query)
+    listed = asyncio.run(fetch_page(config, search_query, limit, start_index))
+    if as_json:
+        print_json([asdict(record) for record in listed])
+        return
+    console = stdout_console()
+    if not listed:
+        console.print(f"No entries at offset {start_index}.")
+        return
+    table = Table("id", "status", "title")
+    for record in listed:
+        status = "[yellow]processed[/]" if record.processed else "[green]new[/]"
+        table.add_row(record.record_id, status, escape(record.title))
+    console.print(table)
+
+
+async def fetch_page(config: CliConfig, query: str, limit: int, start_index: int) -> list[ListedRecord]:
+    processed_ids, _metadata = await assembly.read_state(config)
+    http_client = assembly.build_http_client(config)
+    try:
+        extractor = assembly.build_extractor(config, http_client, discard)
+        raw_listing = await extractor.search(query, limit, start_index)
+        records, _entries = extractor.parse_listing(raw_listing, set())
+    finally:
+        await http_client.aclose()
+    return [
+        ListedRecord(record.record_id, record.title, record.source_url, record.record_id in processed_ids)
+        for record in records
+    ]

sci_etl_cli/commands/status.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import asyncio
+import csv
+from pathlib import Path
+from typing import Any
+
+import click
+from rich.markup import escape
+from rich.table import Table
+
+from sci_etl_cli import assembly
+from sci_etl_cli.errors import CliFailure, ExitCode
+from sci_etl_cli.options import config_argument
+from sci_etl_cli.output import print_json, stdout_console
+from sci_etl_cli.settings import CliConfig, load_cli_config
+
+
+@click.command("status", short_help="Show processed records, saved offset and export rows.")
+@config_argument
+@click.option("--json", "as_json", is_flag=True, help="Print JSON instead of a table.")
+def status_command(config_path: Path, as_json: bool) -> None:
+    """Show CONFIG's progress: processed records, saved offset, last run and export rows.
+
+    Reads state without creating it, so this is safe before the first run.
+    """
+    config = load_cli_config(config_path)
+    processed_ids, metadata = asyncio.run(assembly.read_state(config))
+    report: dict[str, Any] = {
+        "config": str(config_path.resolve()),
+        "backend": config.state.backend,
+        "state": [str(path) for path in state_paths(config)],
+        "processed": len(processed_ids),
+        "saved_offset": metadata.last_start_index,
+        "last_run": metadata.last_run_at,
+        "export": str(config.export.destination),
+        "export_rows": count_rows(config.export.destination),
+    }
+    if as_json:
+        print_json(report)
+        return
+
+    rows = report["export_rows"]
+    table = Table(show_header=False, box=None, pad_edge=False)
+    table.add_column(no_wrap=True)
+    table.add_column(overflow="fold")
+    table.add_row("config", escape(report["config"]))
+    table.add_row("backend", report["backend"])
+    table.add_row("state", escape("\n".join(report["state"])))
+    table.add_row("processed", f"{report['processed']} records")
+    table.add_row("saved offset", str(report["saved_offset"]))
+    table.add_row("last run", escape(report["last_run"] or "never"))
+    table.add_row("export", escape(report["export"]))
+    table.add_row("rows", "not created yet" if rows is None else str(rows))
+    stdout_console().print(table)
+
+
+def state_paths(config: CliConfig) -> tuple[Path, ...]:
+    if config.state.backend == "sqlite":
+        return (config.state.database,)
+    return (config.state.processed_ids, config.state.metadata)
+
+
+def count_rows(path: Path) -> int | None:
+    """Count data rows in the export CSV, or return ``None`` when it does not exist yet.
+
+    Raises:
+        CliFailure: The file exists but cannot be read as UTF-8 CSV.
+    """
+    if not path.is_file():
+        return None
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            return max(sum(1 for _row in csv.reader(handle)) - 1, 0)
+    except (OSError, UnicodeDecodeError, csv.Error) as exc:
+        raise CliFailure(f"Export file could not be read: {path}: {exc}", ExitCode.FAILURE) from exc