yttools 0.1.0__py3-none-any.whl

yttools/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025 William Nichols and YTtools contributors
+"""YTtools: a local-first toolkit for searching public YouTube transcripts."""
+
+from yttools.version import __version__
+
+__all__ = ["__version__"]

yttools/__main__.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025 William Nichols and YTtools contributors
+"""``python -m yttools`` entry point.
+
+Running the module with no arguments starts the web server, matching the
+zero-config experience documented in the README. The installed ``yttools``
+console script routes through :func:`main` and shows help when run bare.
+"""
+
+from __future__ import annotations
+
+import sys
+
+from yttools.cli import app, main
+
+
+def _run_module() -> None:
+    if len(sys.argv) == 1:
+        sys.argv.append("serve")
+    app()
+
+
+if __name__ == "__main__":
+    _run_module()
+else:  # pragma: no cover - re-export for the console-script entry point
+    __all__ = ["main"]

yttools/cli.py

@@ -0,0 +1,237 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025 William Nichols and YTtools contributors
+"""Typer command-line interface.
+
+Every web UI action has a CLI equivalent so the tool can be scripted. Commands are
+added per release; v0.1.0 ships ``fetch``, ``search``, ``list``, ``serve``,
+``config``, ``db``, and ``version``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import typer
+
+from yttools import config as config_module
+from yttools.config import load_settings
+from yttools.core.db import Database
+from yttools.version import __version__
+
+app = typer.Typer(
+    name="yttools",
+    help="Local-first toolkit for searching public YouTube transcripts.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+config_app = typer.Typer(help="Read and write configuration values.", no_args_is_help=True)
+app.add_typer(config_app, name="config")
+
+db_app = typer.Typer(help="Database maintenance commands.", no_args_is_help=True)
+app.add_typer(db_app, name="db")
+
+
+def _open_db() -> Database:
+    return Database.open(load_settings().db_path)
+
+
+@app.command()
+def version() -> None:
+    """Print the installed version."""
+    typer.echo(__version__)
+
+
+@config_app.command("get")
+def config_get(
+    key: str = typer.Argument(..., help="Dotted key, e.g. llm.default_provider"),
+) -> None:
+    """Print a configuration value."""
+    try:
+        value = config_module.get_config_value(key)
+    except KeyError:
+        typer.echo(f"Unknown config key: {key}", err=True)
+        raise typer.Exit(code=1) from None
+    typer.echo(str(value))
+
+
+@config_app.command("set")
+def config_set(
+    key: str = typer.Argument(..., help="Dotted key, e.g. llm.default_provider"),
+    value: str = typer.Argument(..., help="New value"),
+) -> None:
+    """Set a configuration value and persist it to config.toml."""
+    config_module.set_config_value(key, value)
+    typer.echo(f"Set {key} = {value}")
+
+
+@app.command()
+def fetch(
+    urls: list[str] = typer.Argument(..., help="Channel, playlist, or video URLs."),
+    no_transcripts: bool = typer.Option(False, "--no-transcripts", help="Metadata only."),
+    refresh: bool = typer.Option(False, "--refresh", help="Re-fetch even if already stored."),
+    lang: list[str] = typer.Option(["en"], "--lang", help="Preferred caption languages."),
+) -> None:
+    """Download transcripts and metadata for one or more YouTube URLs."""
+    from yttools.tools.fetch import FetchConfig, FetchJob
+
+    config = FetchConfig(
+        include_transcripts=not no_transcripts, languages=lang, force_refresh=refresh
+    )
+    settings = load_settings()
+
+    async def runner() -> None:
+        database = _open_db()
+        from yttools.core.progress import get_bus
+
+        bus = get_bus()
+        job = FetchJob(database, urls, config, bus=bus, captions_dir=settings.home_dir / "captions")
+        queue = await bus.subscribe(job.job_id)
+        task = asyncio.ensure_future(job.run())
+        while True:
+            event = await queue.get()
+            if event is None:
+                break
+            if event.event == "video_update":
+                data = event.data
+                title = data.get("title") or ""
+                typer.echo(f"[{data.get('state'):>17}] {data.get('video_id')}  {title}")
+        summary = await task
+        typer.echo(
+            f"\nDone: {summary.done}  Skipped: {summary.skipped}  "
+            f"No captions: {summary.no_captions}  Errors: {summary.errors}"
+        )
+        database.close()
+
+    asyncio.run(runner())
+
+
+@app.command()
+def search(
+    query: str = typer.Argument(..., help="Search query (phrase, boolean, or prefix syntax)."),
+    channel: list[str] = typer.Option([], "--channel", help="Restrict to channel id(s)."),
+    limit: int = typer.Option(50, "--limit", help="Maximum results to return."),
+    json_output: bool = typer.Option(False, "--json", help="Emit results as JSON."),
+) -> None:
+    """Search transcripts and print ranked matches with timestamp links."""
+    from yttools.tools.search import SearchError, SearchFilters
+    from yttools.tools.search import search as run_search
+
+    database = _open_db()
+    try:
+        response = run_search(
+            database, query, filters=SearchFilters(channel_ids=channel), limit=limit
+        )
+    except SearchError as error:
+        typer.echo(str(error), err=True)
+        raise typer.Exit(code=1) from None
+    finally:
+        database.close()
+
+    if json_output:
+        typer.echo(response.model_dump_json(indent=2))
+        return
+    typer.echo(f"{response.total} result(s) for {query!r}\n")
+    for result in response.results:
+        typer.echo(f"{result.title}")
+        typer.echo(f"  {result.url}")
+        typer.echo(f"  {result.snippet}\n")
+
+
+@app.command("list")
+def list_items(
+    kind: str = typer.Argument(..., help="channels, playlists, or videos."),
+    channel: str | None = typer.Option(None, "--channel", help="Filter videos by channel id."),
+) -> None:
+    """List stored channels, playlists, or videos."""
+    database = _open_db()
+    try:
+        if kind == "channels":
+            for row in database.list_channels():
+                typer.echo(f"{row.id}\t{row.title}")
+        elif kind == "playlists":
+            for playlist in database.list_playlists():
+                typer.echo(f"{playlist.id}\t{playlist.title}")
+        elif kind == "videos":
+            for video in database.list_videos(channel):
+                typer.echo(f"{video.id}\t{video.title}")
+        else:
+            typer.echo("kind must be one of: channels, playlists, videos", err=True)
+            raise typer.Exit(code=1)
+    finally:
+        database.close()
+
+
+@app.command()
+def serve(
+    host: str | None = typer.Option(None, "--host", help="Bind address."),
+    port: int | None = typer.Option(None, "--port", help="Bind port."),
+    no_browser: bool = typer.Option(False, "--no-browser", help="Do not open a browser."),
+    reload: bool = typer.Option(False, "--reload", help="Auto-reload on code changes (dev)."),
+) -> None:
+    """Start the local web UI."""
+    import uvicorn
+
+    from yttools.web.app import open_browser_when_ready
+
+    settings = load_settings()
+    bind_host = host or settings.server.host
+    bind_port = port or settings.server.port
+    if settings.server.open_browser and not no_browser:
+        open_browser_when_ready(f"http://{bind_host}:{bind_port}")
+    uvicorn.run(
+        "yttools.web.app:create_app",
+        factory=True,
+        host=bind_host,
+        port=bind_port,
+        reload=reload,
+    )
+
+
+@db_app.command("migrate")
+def db_migrate() -> None:
+    """Apply any unapplied database migrations."""
+    database = _open_db()
+    applied = database.migrate()
+    database.close()
+    typer.echo(f"Applied {len(applied)} migration(s).")
+
+
+@db_app.command("backup")
+def db_backup() -> None:
+    """Write a timestamped copy of the database file."""
+    import shutil
+    from datetime import UTC, datetime
+
+    settings = load_settings()
+    source = settings.db_path
+    if not source.exists():
+        typer.echo("No database to back up yet.", err=True)
+        raise typer.Exit(code=1)
+    database = _open_db()
+    database._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+    database.close()
+    stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+    target = source.with_name(f"yttools.backup-{stamp}.db")
+    shutil.copy2(source, target)
+    typer.echo(f"Backed up to {target}")
+
+
+@db_app.command("reset")
+def db_reset(
+    yes: bool = typer.Option(False, "--yes", help="Skip the confirmation prompt."),
+) -> None:
+    """Delete the database and recreate an empty schema."""
+    settings = load_settings()
+    if not yes:
+        typer.confirm(f"This deletes {settings.db_path} and all stored data. Continue?", abort=True)
+    for suffix in ("", "-wal", "-shm"):
+        candidate = settings.db_path.with_name(settings.db_path.name + suffix)
+        candidate.unlink(missing_ok=True)
+    _open_db().close()
+    typer.echo("Database reset.")
+
+
+def main() -> None:
+    """Console-script entry point."""
+    app()

yttools/config.py

@@ -0,0 +1,216 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025 William Nichols and YTtools contributors
+"""Application configuration.
+
+Settings load from ``$YTTOOLS_HOME/config.toml`` (default ``~/.yttools``). Hosted
+provider API keys fall back to environment variables when the config value is
+empty, in this resolution order: config value, then environment variable, then
+empty (which leaves the provider disabled).
+"""
+
+from __future__ import annotations
+
+import os
+import tomllib
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+DEFAULT_HOME = "~/.yttools"
+
+# Hosted-provider API keys are also read from these environment variables.
+API_KEY_ENV_VARS: dict[str, str] = {
+    "anthropic": "ANTHROPIC_API_KEY",
+    "openai": "OPENAI_API_KEY",
+    "gemini": "GEMINI_API_KEY",
+}
+
+
+class PathsConfig(BaseModel):
+    home: str = DEFAULT_HOME
+
+
+class FetchConfig(BaseModel):
+    concurrent_videos: int = 3
+    preferred_caption_lang: str = "en"
+
+
+class OllamaConfig(BaseModel):
+    base_url: str = "http://localhost:11434"
+
+
+class HostedProviderConfig(BaseModel):
+    api_key: str = ""
+    default_model: str
+
+
+class LLMConfig(BaseModel):
+    default_provider: str = "ollama"
+    default_model: str = "llama3.1:8b"
+    concurrent_requests: int = 2
+    embedding_model: str = "nomic-embed-text"
+    ollama: OllamaConfig = Field(default_factory=OllamaConfig)
+    anthropic: HostedProviderConfig = Field(
+        default_factory=lambda: HostedProviderConfig(default_model="claude-sonnet-4-5")
+    )
+    openai: HostedProviderConfig = Field(
+        default_factory=lambda: HostedProviderConfig(default_model="gpt-4o")
+    )
+    gemini: HostedProviderConfig = Field(
+        default_factory=lambda: HostedProviderConfig(default_model="gemini-2.0-flash")
+    )
+
+
+class ServerConfig(BaseModel):
+    host: str = "127.0.0.1"
+    port: int = 8765
+    open_browser: bool = True
+
+
+class Settings(BaseModel):
+    paths: PathsConfig = Field(default_factory=PathsConfig)
+    fetch: FetchConfig = Field(default_factory=FetchConfig)
+    llm: LLMConfig = Field(default_factory=LLMConfig)
+    server: ServerConfig = Field(default_factory=ServerConfig)
+
+    @property
+    def home_dir(self) -> Path:
+        return Path(self.paths.home).expanduser()
+
+    @property
+    def db_path(self) -> Path:
+        return self.home_dir / "yttools.db"
+
+    @property
+    def config_path(self) -> Path:
+        return self.home_dir / "config.toml"
+
+    @property
+    def exports_dir(self) -> Path:
+        return self.home_dir / "exports"
+
+
+def resolve_home(home: str | Path | None = None) -> Path:
+    """Resolve the data directory, honoring the ``YTTOOLS_HOME`` environment variable."""
+    if home is not None:
+        return Path(home).expanduser()
+    env_home = os.environ.get("YTTOOLS_HOME")
+    return Path(env_home).expanduser() if env_home else Path(DEFAULT_HOME).expanduser()
+
+
+def read_raw_config(home: str | Path | None = None) -> dict[str, Any]:
+    """Read the raw config TOML into a dict, or return an empty dict if absent."""
+    config_path = resolve_home(home) / "config.toml"
+    if not config_path.exists():
+        return {}
+    with config_path.open("rb") as handle:
+        return tomllib.load(handle)
+
+
+def _apply_env_key_overrides(settings: Settings) -> None:
+    """Fill empty hosted-provider keys from environment variables."""
+    for provider, env_var in API_KEY_ENV_VARS.items():
+        provider_config: HostedProviderConfig = getattr(settings.llm, provider)
+        if not provider_config.api_key:
+            env_value = os.environ.get(env_var, "")
+            if env_value:
+                provider_config.api_key = env_value
+
+
+def load_settings(home: str | Path | None = None) -> Settings:
+    """Load settings from disk and apply environment-variable overrides."""
+    resolved_home = resolve_home(home)
+    raw = read_raw_config(resolved_home)
+    settings = Settings.model_validate(raw)
+    settings.paths.home = str(resolved_home)
+    _apply_env_key_overrides(settings)
+    return settings
+
+
+def _toml_scalar(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    escaped = str(value).replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def dumps_toml(data: dict[str, Any]) -> str:
+    """Serialize a nested config dict to TOML.
+
+    Handles the flat-and-nested-tables shape this project's config uses: top-level
+    scalars, single tables, and one level of nested tables.
+    """
+    lines: list[str] = []
+    nested: list[tuple[str, dict[str, Any]]] = []
+    for key, value in data.items():
+        if isinstance(value, dict):
+            nested.append((key, value))
+        else:
+            lines.append(f"{key} = {_toml_scalar(value)}")
+    for table, table_value in nested:
+        sub_tables: list[tuple[str, dict[str, Any]]] = []
+        lines.append("")
+        lines.append(f"[{table}]")
+        for key, value in table_value.items():
+            if isinstance(value, dict):
+                sub_tables.append((key, value))
+            else:
+                lines.append(f"{key} = {_toml_scalar(value)}")
+        for sub_table, sub_value in sub_tables:
+            lines.append("")
+            lines.append(f"[{table}.{sub_table}]")
+            for key, value in sub_value.items():
+                lines.append(f"{key} = {_toml_scalar(value)}")
+    return "\n".join(lines).strip() + "\n"
+
+
+def write_settings(settings: Settings, home: str | Path | None = None) -> Path:
+    """Persist settings to ``config.toml``, creating the data directory if needed."""
+    resolved_home = resolve_home(home)
+    resolved_home.mkdir(parents=True, exist_ok=True)
+    config_path = resolved_home / "config.toml"
+    payload = settings.model_dump()
+    payload["paths"]["home"] = DEFAULT_HOME
+    config_path.write_text(dumps_toml(payload), encoding="utf-8")
+    return config_path
+
+
+def get_config_value(key: str, home: str | Path | None = None) -> Any:
+    """Read a dotted config key (for example ``llm.default_provider``)."""
+    settings = load_settings(home)
+    current: Any = settings.model_dump()
+    for part in key.split("."):
+        if not isinstance(current, dict) or part not in current:
+            raise KeyError(key)
+        current = current[part]
+    return current
+
+
+def set_config_value(key: str, value: str, home: str | Path | None = None) -> Settings:
+    """Set a dotted config key and persist. Values are coerced to match the schema."""
+    raw = read_raw_config(home)
+    parts = key.split(".")
+    cursor = raw
+    for part in parts[:-1]:
+        existing = cursor.get(part)
+        if not isinstance(existing, dict):
+            existing = {}
+            cursor[part] = existing
+        cursor = existing
+    cursor[parts[-1]] = _coerce_value(value)
+    settings = Settings.model_validate(raw)
+    write_settings(settings, home)
+    return settings
+
+
+def _coerce_value(value: str) -> Any:
+    lowered = value.lower()
+    if lowered in {"true", "false"}:
+        return lowered == "true"
+    try:
+        return int(value)
+    except ValueError:
+        return value

yttools/core/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# Copyright (C) 2025 William Nichols and YTtools contributors
+"""Shared infrastructure: database, models, YouTube access, transcripts, LLM."""