subpeeko 0.1.0__py3-none-any.whl

subpeeko/__init__.py

@@ -0,0 +1,3 @@
+"""Create English-learning subtitles from bilingual ASS files."""
+
+__version__ = "0.1.0"

subpeeko/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

subpeeko/checkpoint.py

@@ -0,0 +1,88 @@
+"""Local checkpoints for model results already reviewed for an exact input."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sqlite3
+import sys
+from pathlib import Path
+from threading import Lock
+
+from . import model, subtitles, vocabulary
+from .config import Config
+
+
+class CheckpointError(Exception):
+    pass
+
+
+def cache_dir() -> Path:
+    if sys.platform == "win32":
+        root = Path(os.environ.get("LOCALAPPDATA") or Path.home() / "AppData" / "Local")
+    else:
+        root = Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache")
+        if not root.is_absolute():
+            root = Path.home() / ".cache"
+    return root / "subpeeko" / "checkpoints"
+
+
+def checkpoint_key(path: Path, source_hash: bytes, config: Config, level: str, *, batch_size: int,
+                   thinking: str, max_output_tokens: int) -> str:
+    digest = hashlib.sha256()
+    try:
+        values = (
+            b"subpeeko-checkpoint-v1", str(path.resolve()).encode(), source_hash,
+            config.base_url.encode(), config.model_name.encode(), config.api_key.encode(),
+            level.encode(), str(batch_size).encode(), thinking.encode(), str(max_output_tokens).encode(),
+            Path(model.__file__).read_bytes(), Path(subtitles.__file__).read_bytes(),
+            Path(vocabulary.__file__).read_bytes(),
+            vocabulary.files("subpeeko").joinpath("data/vocabulary.json").read_bytes(),
+        )
+    except OSError as exc:
+        raise CheckpointError("Cannot read checkpoint inputs") from exc
+    for value in values:
+        digest.update(len(value).to_bytes(8, "big"))
+        digest.update(value)
+    return digest.hexdigest()
+
+
+class Checkpoint:
+    def __init__(self, key: str):
+        self.lock = Lock()
+        self.connection = None
+        try:
+            root = cache_dir()
+            root.mkdir(mode=0o700, parents=True, exist_ok=True)
+            path = root / f"{key}.sqlite3"
+            descriptor = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
+            os.close(descriptor)
+            self.connection = sqlite3.connect(path, timeout=30, check_same_thread=False)
+            self.connection.execute("CREATE TABLE IF NOT EXISTS notes (id INTEGER PRIMARY KEY, note TEXT NOT NULL)")
+            self.connection.commit()
+        except (OSError, sqlite3.Error) as exc:
+            if self.connection is not None:
+                self.connection.close()
+            raise CheckpointError("Cannot open the local checkpoint; use --no-resume to disable it") from exc
+
+    def load(self, valid_ids: set[int]) -> dict[int, str]:
+        try:
+            with self.lock:
+                rows = self.connection.execute("SELECT id, note FROM notes").fetchall()
+            if any(type(identifier) is not int or identifier not in valid_ids or not isinstance(note, str)
+                   or any(char in note for char in "{}\\\r\n") for identifier, note in rows):
+                raise CheckpointError("Local checkpoint contains invalid results; use --no-resume")
+            return dict(rows)
+        except sqlite3.Error as exc:
+            raise CheckpointError("Cannot read the local checkpoint; use --no-resume") from exc
+
+    def save(self, notes: dict[int, str]) -> None:
+        try:
+            with self.lock, self.connection:
+                self.connection.executemany("INSERT OR REPLACE INTO notes (id, note) VALUES (?, ?)", notes.items())
+        except sqlite3.Error as exc:
+            raise CheckpointError("Cannot save the local checkpoint; completed batches remain available") from exc
+
+    def close(self) -> None:
+        if self.connection is not None:
+            self.connection.close()

subpeeko/cli.py

@@ -0,0 +1,266 @@
+"""Command-line entry point for SubPeeko."""
+
+from __future__ import annotations
+
+import argparse
+import math
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+from . import __version__
+from .checkpoint import Checkpoint, CheckpointError, checkpoint_key
+from .config import ConfigError, configure, load_config, show_config
+from .model import BATCH_SIZE, DEFAULT_CONCURRENCY, DEFAULT_RPM, DEFAULT_TPM, ModelClient, ModelError
+from .subtitles import AssFile, SubtitleError
+from .vocabulary import LEVELS
+
+OUTPUT_SUFFIX = ".subpeeko.ass"
+
+
+def _batch_size(value: str) -> int:
+    try:
+        number = int(value)
+        if 1 <= number <= 100:
+            return number
+    except ValueError:
+        pass
+    raise argparse.ArgumentTypeError("batch size must be an integer between 1 and 100")
+
+
+def _positive_int(value: str) -> int:
+    try:
+        number = int(value)
+        if number > 0:
+            return number
+    except ValueError:
+        pass
+    raise argparse.ArgumentTypeError("must be a positive integer")
+
+
+def _positive_float(value: str) -> float:
+    try:
+        number = float(value)
+        if math.isfinite(number) and number > 0:
+            return number
+    except ValueError:
+        pass
+    raise argparse.ArgumentTypeError("must be a positive finite number")
+
+
+def _nonnegative_int(value: str) -> int:
+    try:
+        number = int(value)
+        if number >= 0:
+            return number
+    except ValueError:
+        pass
+    raise argparse.ArgumentTypeError("must be a nonnegative integer")
+
+
+def parser() -> argparse.ArgumentParser:
+    result = argparse.ArgumentParser(
+        prog="subpeeko",
+        description="Create English-learning subtitles from bilingual ASS files.",
+        epilog="Run 'subpeeko config' to configure the model API."
+    )
+    result.add_argument("input", type=Path, help="an ASS file or a directory of ASS files")
+    result.add_argument("--level", choices=LEVELS, required=True, help="the learner's English level")
+    result.add_argument("--recursive", action="store_true", help="include subdirectories")
+    result.add_argument("--overwrite", action="store_true", help="replace existing generated output")
+    result.add_argument("--dry-run", action="store_true", help="list input and output paths without calling the model")
+    result.add_argument("--batch-size", type=_batch_size, default=BATCH_SIZE, help=f"dialogues per model request (default: {BATCH_SIZE})")
+    result.add_argument("--concurrency", type=_positive_int, default=DEFAULT_CONCURRENCY, help=f"parallel model requests (default: {DEFAULT_CONCURRENCY})")
+    result.add_argument("--rpm", type=_positive_int, default=DEFAULT_RPM, help=f"available requests per minute for this process (default: {DEFAULT_RPM})")
+    result.add_argument("--tpm", type=_positive_int, default=DEFAULT_TPM, help=f"available input and output tokens per minute (default: {DEFAULT_TPM})")
+    result.add_argument("--qps", type=_positive_float, help="maximum requests per second; defaults to RPM / 60")
+    result.add_argument("--max-output-tokens", type=_positive_int, default=2048, help="output token reservation when rate limiting (default: 2048)")
+    result.add_argument("--thinking", choices=("off", "on", "auto"), default="off", help="model thinking mode (default: off; auto omits the provider-specific setting)")
+    result.add_argument("--reasoning-reserve", type=_nonnegative_int, default=8192, help="additional reasoning-token reservation for providers without a combined cap (default: 8192)")
+    result.add_argument("--no-resume", action="store_true", help="do not read or write local checkpoints")
+    result.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return result
+
+
+def _config_command(argv: list[str]) -> int:
+    p = argparse.ArgumentParser(prog="subpeeko config", description="Manage model API configuration.")
+    p.add_argument("action", nargs="?", choices=("set", "show", "test"), default="set")
+    args = p.parse_args(argv)
+    if args.action == "set":
+        configure()
+    elif args.action == "show":
+        show_config()
+    else:
+        client = ModelClient(load_config())
+        client.test_connection()
+        print("Model API connection succeeded.")
+    return 0
+
+
+def input_files(path: Path, recursive: bool) -> list[Path]:
+    if path.is_file():
+        if path.suffix.casefold() != ".ass":
+            raise SubtitleError("Input file must have an .ass extension")
+        if path.name.casefold().endswith(OUTPUT_SUFFIX):
+            raise SubtitleError("Generated SubPeeko subtitles cannot be used as input")
+        return [path]
+    if not path.is_dir():
+        raise SubtitleError(f"Input path does not exist: {path}")
+    items = path.rglob("*") if recursive else path.iterdir()
+    try:
+        return sorted(
+            (item for item in items if item.is_file() and item.suffix.casefold() == ".ass"
+             and not item.name.casefold().endswith(OUTPUT_SUFFIX)),
+            key=lambda item: str(item).casefold(),
+        )
+    except OSError as exc:
+        raise SubtitleError(f"Cannot read directory: {path}") from exc
+
+
+def output_path(path: Path, level: str) -> Path:
+    return path.with_name(f"{path.stem}.{level}{OUTPUT_SUFFIX}")
+
+
+def _write_output(path: Path, data: bytes, overwrite: bool) -> None:
+    if not overwrite:
+        try:
+            with path.open("xb") as destination:
+                try:
+                    destination.write(data)
+                except OSError:
+                    destination.close()
+                    path.unlink(missing_ok=True)
+                    raise
+        except FileExistsError as exc:
+            raise SubtitleError(f"Output already exists: {path} (use --overwrite to replace it)") from exc
+        except OSError as exc:
+            raise SubtitleError(f"Cannot write output: {path}") from exc
+        return
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(dir=path.parent, prefix=".subpeeko-", suffix=".tmp", delete=False) as stream:
+            temporary = Path(stream.name)
+            stream.write(data)
+        os.replace(temporary, path)
+    except OSError as exc:
+        raise SubtitleError(f"Cannot write output: {path}") from exc
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def _report_request(event: dict) -> None:
+    usage = event["usage"] or {}
+    def count(name):
+        value = usage.get(name)
+        return str(value) if type(value) is int and value >= 0 else "?"
+    print(
+        f"  Request {event['id']}: targets={event['targets']}, status={event['status']}, "
+        f"network={event['request_seconds']:.2f}s, wait={event['wait_seconds']:.2f}s, "
+        f"tokens in/out/reasoning={count('prompt_tokens')}/{count('completion_tokens')}/{count('reasoning_tokens')}",
+        flush=True,
+    )
+
+
+def _convert(args: argparse.Namespace) -> int:
+    clients = []
+    started = time.monotonic()
+    try:
+        return _convert_files(args, clients)
+    finally:
+        if clients:
+            client = clients[0]
+            def total(key):
+                count = client.usage_counts[key]
+                if not count:
+                    return "?"
+                value = str(client.usage[key])
+                return value if count == client.usage_responses else f"{value} (partial {count}/{client.usage_responses})"
+            print(f"Elapsed: {time.monotonic() - started:.2f}s. Usage: input={total('prompt_tokens')}, output={total('completion_tokens')}, "
+                  f"reasoning={total('reasoning_tokens')}; usage reported for {client.usage_responses}/{client.requests} requests; "
+                  f"{client.retries} retry/split event(s).")
+
+
+def _convert_files(args: argparse.Namespace, clients: list[ModelClient]) -> int:
+    paths = input_files(args.input, args.recursive)
+    if not paths:
+        print("No input ASS files found.")
+        return 0
+    if args.dry_run:
+        for path in paths:
+            print(f"{path} -> {output_path(path, args.level)}")
+        print(f"{len(paths)} file(s); no changes made and no model requests sent.")
+        return 0
+
+    client = None
+    succeeded = failed = 0
+    for path in paths:
+        output = output_path(path, args.level)
+        checkpoint = None
+        try:
+            if output.exists() and not args.overwrite:
+                raise SubtitleError(f"Output already exists: {output} (use --overwrite to replace it)")
+            subtitles = AssFile.read(path)
+            if not subtitles.dialogues:
+                raise SubtitleError("No supported bilingual dialogue found")
+            if client is None:
+                client = ModelClient(
+                    load_config(), batch_size=args.batch_size, workers=args.concurrency,
+                    rpm=args.rpm, tpm=args.tpm, qps=args.qps, max_output_tokens=args.max_output_tokens,
+                    thinking=args.thinking, reasoning_reserve=args.reasoning_reserve, on_request=_report_request,
+                )
+                clients.append(client)
+            cached = {}
+            if not args.no_resume:
+                key = checkpoint_key(path, subtitles.source_hash, client.config, args.level,
+                                     batch_size=args.batch_size, thinking=client.thinking,
+                                     max_output_tokens=args.max_output_tokens)
+                checkpoint = Checkpoint(key)
+                cached = checkpoint.load({item.id for item in subtitles.dialogues})
+                if cached:
+                    print(f"Resuming: {len(cached)}/{len(subtitles.dialogues)} dialogue(s) from local checkpoint.")
+            print(f"Processing: {path} ({len(subtitles.dialogues)} bilingual dialogue(s))", flush=True)
+            notes = client.analyze(
+                subtitles.dialogues, args.level,
+                progress=lambda done, total: print(f"  Reviewed {done}/{total} dialogue(s).", flush=True),
+                on_retry=lambda message: print(f"  {message}", file=sys.stderr, flush=True),
+                initial_results=cached, on_batch=checkpoint.save if checkpoint else None,
+            )
+            result = subtitles.render(notes)
+            _write_output(output, result, args.overwrite)
+            succeeded += 1
+            print(f"Created: {output} ({len(subtitles.dialogues)} bilingual dialogue(s))")
+            if any(len(note) > 90 for note in notes.values()):
+                print("  Some annotations may be too long for the available display time.")
+        except (CheckpointError, ConfigError, ModelError, SubtitleError) as exc:
+            failed += 1
+            print(f"Failed: {path}: {exc}", file=sys.stderr)
+        finally:
+            if checkpoint is not None:
+                checkpoint.close()
+    print(f"Finished: {succeeded} created, {failed} failed; {client.requests if client else 0} model request(s).")
+    return 1 if failed else 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    # Redirected Windows output may use an encoding that cannot represent paths.
+    for stream in (sys.stdout, sys.stderr):
+        if hasattr(stream, "reconfigure"):
+            stream.reconfigure(errors="backslashreplace")
+    argv = list(sys.argv[1:] if argv is None else argv)
+    try:
+        if argv and argv[0] == "config":
+            return _config_command(argv[1:])
+        p = parser()
+        args = p.parse_args(argv)
+        if args.concurrency > 32:
+            p.error("--concurrency must not exceed 32")
+        return _convert(args)
+    except (CheckpointError, ConfigError, ModelError, SubtitleError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 1
+    except (KeyboardInterrupt, EOFError):
+        print("Interrupted.", file=sys.stderr)
+        return 130

subpeeko/config.py

@@ -0,0 +1,171 @@
+"""Local configuration and system credential storage."""
+
+from __future__ import annotations
+
+import getpass
+import json
+import os
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from dotenv import dotenv_values
+
+SERVICE = "subpeeko"
+KEY_NAME = "model-api-key"
+SECURE_BACKENDS = {
+    "keyring.backends.Windows",
+    "keyring.backends.SecretService",
+    "keyring.backends.kwallet",
+    "keyring.backends.macOS",
+}
+
+
+class ConfigError(Exception):
+    pass
+
+
+@dataclass(frozen=True)
+class Config:
+    base_url: str = ""
+    model_name: str = ""
+    api_key: str = field(default="", repr=False)
+
+
+def config_path() -> Path:
+    if sys.platform == "win32":
+        root = Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming")
+    elif sys.platform == "darwin":
+        root = Path.home() / "Library" / "Application Support"
+    else:
+        root = Path(os.environ.get("XDG_CONFIG_HOME") or Path.home() / ".config")
+        if not root.is_absolute():
+            root = Path.home() / ".config"
+    return root / SERVICE / "config.json"
+
+
+def _read_saved() -> dict[str, str]:
+    path = config_path()
+    if not path.exists():
+        return {}
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        raise ConfigError(f"Cannot read configuration: {path}") from exc
+    if not isinstance(data, dict):
+        raise ConfigError(f"Invalid configuration: {path}")
+    return {key: value for key, value in data.items() if isinstance(value, str)}
+
+
+def _keyring():
+    try:
+        import keyring
+
+        backend = keyring.get_keyring()
+        if type(backend).__module__ == "keyring.backends.chainer":
+            backend = next((item for item in backend.backends if type(item).__module__ in SECURE_BACKENDS), None)
+            if backend is None:
+                return None
+        if type(backend).__module__ not in SECURE_BACKENDS:
+            return None
+        return backend
+    except Exception:
+        return None
+
+
+def _stored_key() -> str:
+    backend = _keyring()
+    if backend is None:
+        return ""
+    try:
+        return backend.get_password(SERVICE, KEY_NAME) or ""
+    except Exception:
+        return ""
+
+
+def load_config(require: bool = True) -> Config:
+    saved = _read_saved()
+    try:
+        dotenv = dotenv_values(Path.cwd() / ".env")
+    except OSError as exc:
+        raise ConfigError("Cannot read the local .env file") from exc
+
+    def value(name: str, default: str = "") -> str:
+        return (os.environ.get(name) or dotenv.get(name) or default).strip()
+
+    config = Config(
+        base_url=value("SUBPEEKO_BASE_URL", saved.get("base_url", "")),
+        model_name=value("SUBPEEKO_MODEL_NAME", saved.get("model_name", "")),
+        api_key=value("SUBPEEKO_API_KEY") or _stored_key(),
+    )
+    if require:
+        missing = [
+            name for name, item in (
+                ("base URL", config.base_url),
+                ("model name", config.model_name),
+                ("API key", config.api_key),
+            ) if not item
+        ]
+        if missing:
+            raise ConfigError(
+                "Missing " + ", ".join(missing)
+                + ". Run 'subpeeko config' or set the SUBPEEKO_* environment variables."
+            )
+        validate_base_url(config.base_url)
+    return config
+
+
+def validate_base_url(url: str) -> None:
+    try:
+        parsed = urlsplit(url)
+        valid = (parsed.scheme in {"http", "https"} and parsed.hostname
+                 and not parsed.username and not parsed.password and not parsed.query and not parsed.fragment)
+    except ValueError:
+        valid = False
+    if not valid:
+        raise ConfigError("Base URL must be an HTTP(S) API root without credentials or a query string")
+
+
+def configure() -> None:
+    saved = _read_saved()
+    print("Configure an OpenAI-compatible Chat Completions API.")
+    base = input(f"Base URL [{saved.get('base_url', '')}]: ").strip() or saved.get("base_url", "")
+    model = input(f"Model name [{saved.get('model_name', '')}]: ").strip() or saved.get("model_name", "")
+    if not base or not model:
+        raise ConfigError("Both base URL and model name are required")
+    validate_base_url(base)
+
+    path = config_path()
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps({"base_url": base, "model_name": model}, indent=2) + "\n", encoding="utf-8")
+    except OSError as exc:
+        raise ConfigError(f"Cannot save configuration: {path}") from exc
+
+    backend = _keyring()
+    if backend is None:
+        print("No supported system credential store is available; set SUBPEEKO_API_KEY in the environment.")
+    else:
+        if not sys.stdin.isatty():
+            raise ConfigError("An interactive terminal is required to enter a key; use SUBPEEKO_API_KEY otherwise")
+        key = getpass.getpass("API key (leave blank to keep the existing key): ").strip()
+        if key:
+            try:
+                backend.set_password(SERVICE, KEY_NAME, key)
+            except Exception as exc:
+                raise ConfigError("Cannot save the API key in the system credential store; use SUBPEEKO_API_KEY") from exc
+    print(f"Configuration saved to {path}. API keys are never stored in this file.")
+
+
+def show_config() -> None:
+    config = load_config(require=False)
+    base_url = config.base_url or "(not set)"
+    if config.base_url:
+        try:
+            validate_base_url(config.base_url)
+        except ConfigError:
+            base_url = "(invalid URL)"
+    print(f"Base URL: {base_url}")
+    print(f"Model: {config.model_name or '(not set)'}")
+    print(f"API key: {'configured' if config.api_key else '(not set)'}")

subpeeko/data/LICENSE-KyleBing.txt

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2022-2026, KyleBing
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

subpeeko/data/vocabulary-source.json

@@ -0,0 +1,4 @@
+{
+  "repository": "https://github.com/KyleBing/english-vocabulary",
+  "revision": "731240a848dca14e2cf17510f384312b788863c1"
+}