textflowkit 0.1.3__py3-none-any.whl

textflowkit/core/service.py

@@ -0,0 +1,146 @@
+"""Opt-in production profile; localhost owner mode stays unconfined."""
+
+from __future__ import annotations
+
+import math
+import os
+import subprocess
+from pathlib import Path
+
+ENV_PROFILE = "TEXTFLOWKIT_PROFILE"
+ENV_API_TOKEN = "TEXTFLOWKIT_API_TOKEN"
+ENV_WORK_ROOT = "TEXTFLOWKIT_WORK_ROOT"
+ENV_MAX_REQUEST_BYTES = "TEXTFLOWKIT_MAX_REQUEST_BYTES"
+ENV_RATE_PER_MINUTE = "TEXTFLOWKIT_RATE_PER_MINUTE"
+ENV_MAX_DURATION_SECONDS = "TEXTFLOWKIT_MAX_DURATION_SECONDS"
+ENV_MAX_OUTPUT_BYTES = "TEXTFLOWKIT_MAX_OUTPUT_BYTES"
+ENV_MAX_MEDIA_BYTES = "TEXTFLOWKIT_MAX_MEDIA_BYTES"
+ENV_EGRESS_PROXY = "TEXTFLOWKIT_EGRESS_PROXY"
+ENV_FFMPEG_TIMEOUT_SECONDS = "TEXTFLOWKIT_FFMPEG_TIMEOUT_SECONDS"
+
+
+class ServiceConfigurationError(ValueError):
+    """The production profile cannot safely start with this configuration."""
+
+
+def production_enabled() -> bool:
+    value = os.environ.get(ENV_PROFILE, "developer").strip().lower()
+    if value not in {"developer", "production"}:
+        raise ServiceConfigurationError(
+            f"{ENV_PROFILE} must be 'developer' or 'production', not '{value}'"
+        )
+    return value == "production"
+
+
+def positive_limit(name: str, default: int) -> int:
+    raw = os.environ.get(name)
+    try:
+        value = int(raw) if raw else default
+    except ValueError as exc:
+        raise ServiceConfigurationError(f"{name} must be a positive integer") from exc
+    if value <= 0:
+        raise ServiceConfigurationError(f"{name} must be a positive integer")
+    return value
+
+
+def validate_production_config() -> None:
+    """Fail closed before handling any production HTTP request."""
+    if not production_enabled():
+        return
+    from textflowkit.core.paths import ENV_INPUT_ROOT, ENV_OUTPUT_ROOT
+
+    required = (ENV_API_TOKEN, ENV_INPUT_ROOT, ENV_OUTPUT_ROOT, "TEXTFLOWKIT_DB", ENV_WORK_ROOT)
+    missing = [name for name in required if not os.environ.get(name)]
+    if missing:
+        raise ServiceConfigurationError(
+            f"production profile requires: {', '.join(missing)}"
+        )
+    if len(os.environ[ENV_API_TOKEN]) < 16:
+        raise ServiceConfigurationError(f"{ENV_API_TOKEN} must have at least 16 characters")
+    if os.environ["TEXTFLOWKIT_DB"] == ":memory:":
+        raise ServiceConfigurationError("production requires an on-disk TEXTFLOWKIT_DB")
+    from textflowkit.core.executor import get_default_executor
+    from textflowkit.core.jobs import get_default_store
+    from textflowkit.core.sqlite_store import SqliteJobStore
+
+    store = get_default_store()
+    db_path = Path(os.environ["TEXTFLOWKIT_DB"]).expanduser().resolve()
+    if not isinstance(store, SqliteJobStore) or Path(store.path).resolve() != db_path:
+        raise ServiceConfigurationError("production requires the active store to use TEXTFLOWKIT_DB")
+    if get_default_executor().store is not store:
+        raise ServiceConfigurationError("production executor and durable store must match")
+    input_root = Path(os.environ[ENV_INPUT_ROOT]).expanduser().resolve()
+    if not input_root.is_dir():
+        raise ServiceConfigurationError(f"{ENV_INPUT_ROOT} must name an existing directory")
+    for name in (ENV_OUTPUT_ROOT, ENV_WORK_ROOT):
+        path = Path(os.environ[name]).expanduser().resolve()
+        path.mkdir(parents=True, exist_ok=True)
+        if not path.is_dir():
+            raise ServiceConfigurationError(f"{name} must name a directory")
+    positive_limit(ENV_MAX_REQUEST_BYTES, 64 * 1024)
+    positive_limit(ENV_RATE_PER_MINUTE, 60)
+    positive_limit(ENV_MAX_DURATION_SECONDS, 4 * 3600)
+    positive_limit(ENV_MAX_OUTPUT_BYTES, 50 * 1024 * 1024)
+    positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024)
+    positive_limit(ENV_FFMPEG_TIMEOUT_SECONDS, 600)
+    positive_limit("TEXTFLOWKIT_MAX_PENDING_JOBS", 100)
+
+
+def service_work_root() -> str | None:
+    if not production_enabled():
+        return None
+    validate_production_config()
+    return str(Path(os.environ[ENV_WORK_ROOT]).expanduser().resolve())
+
+
+def _probe_duration(media: Path) -> float | None:
+    """Return known duration using a bounded probe; unknown is handled at decode."""
+    from textflowkit.sources.acquire import require_tool
+
+    ffprobe = require_tool("ffprobe")
+    try:
+        proc = subprocess.run(
+            [ffprobe, "-v", "error", "-show_entries", "format=duration",
+             "-of", "default=noprint_wrappers=1:nokey=1", str(media)],
+            capture_output=True, text=True, timeout=30, check=False,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise ServiceConfigurationError("ffprobe timed out before decode") from exc
+    if proc.returncode != 0:
+        return None
+    try:
+        duration = float(proc.stdout.strip())
+    except ValueError:
+        return None
+    return duration if math.isfinite(duration) and duration >= 0 else None
+
+
+def enforce_predecode_limits(media: Path) -> None:
+    """Reject known oversize/overlong media before launching full extraction."""
+    if not production_enabled():
+        return
+    maximum = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024)
+    if media.stat().st_size > maximum:
+        raise ServiceConfigurationError("media exceeds the configured size limit")
+    duration = _probe_duration(media)
+    if duration is not None and duration > positive_limit(ENV_MAX_DURATION_SECONDS, 4 * 3600):
+        raise ServiceConfigurationError("source duration exceeds the configured limit")
+
+
+def enforce_media_limits(media: Path, audio: Path) -> None:
+    """Defense in depth after bounded acquisition and decode."""
+    if not production_enabled():
+        return
+    maximum = positive_limit(ENV_MAX_MEDIA_BYTES, 1024 * 1024 * 1024)
+    if media.stat().st_size > maximum or audio.stat().st_size > maximum:
+        raise ServiceConfigurationError("media exceeds the configured size limit")
+    duration = _probe_duration(audio)
+    if duration is None:
+        raise ServiceConfigurationError("cannot verify source duration with ffprobe")
+    if duration > positive_limit(ENV_MAX_DURATION_SECONDS, 4 * 3600):
+        raise ServiceConfigurationError("source duration exceeds the configured limit")
+
+
+def enforce_output_limit(size: int) -> None:
+    if production_enabled() and size > positive_limit(ENV_MAX_OUTPUT_BYTES, 50 * 1024 * 1024):
+        raise ServiceConfigurationError("rendered output exceeds the configured size limit")

textflowkit/core/sqlite_store.py

@@ -0,0 +1,233 @@
+"""Durable job store backed by SQLite.
+
+Chosen over a hand-rolled file format because SQLite is stdlib, transactional,
+and already safe for concurrent access. One connection guarded by a lock is
+sufficient at this scale and keeps the implementation obvious.
+
+Ordering uses the table's AUTOINCREMENT `seq`, not `created_at` - same reasoning
+as the in-memory store: wall-clock ordering is not portable across platforms.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+from textflowkit.core.jobs import Job, JobState, JobStore
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS jobs (
+    seq              INTEGER PRIMARY KEY AUTOINCREMENT,
+    id               TEXT    NOT NULL UNIQUE,
+    source           TEXT    NOT NULL,
+    state            TEXT    NOT NULL,
+    created_at       REAL    NOT NULL,
+    updated_at       REAL    NOT NULL,
+    progress         TEXT    NOT NULL DEFAULT '',
+    error            TEXT,
+    transcript       TEXT,
+    outputs          TEXT    NOT NULL DEFAULT '[]',
+    cancel_requested INTEGER NOT NULL DEFAULT 0,
+    checkpoint       TEXT,
+    request          TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_jobs_state ON jobs(state);
+"""
+
+# Columns a caller may patch through update().
+_MUTABLE = frozenset(
+    {
+        "state",
+        "progress",
+        "error",
+        "transcript",
+        "outputs",
+        "cancel_requested",
+        "checkpoint",
+        "request",
+    }
+)
+
+_CHECKPOINT_COLUMN = "checkpoint"
+
+
+def _row_to_job(row: sqlite3.Row) -> Job:
+    transcript = json.loads(row["transcript"]) if row["transcript"] else None
+    outputs = json.loads(row["outputs"]) if row["outputs"] else []
+    keys = set(row.keys())
+    raw_checkpoint = row["checkpoint"] if "checkpoint" in keys else None
+    checkpoint = json.loads(raw_checkpoint) if raw_checkpoint else None
+    raw_request = row["request"] if "request" in keys else None
+    request = json.loads(raw_request) if raw_request else None
+    return Job(
+        id=row["id"],
+        source=row["source"],
+        state=JobState(row["state"]),
+        created_at=row["created_at"],
+        updated_at=row["updated_at"],
+        progress=row["progress"] or "",
+        error=row["error"],
+        transcript=transcript,
+        outputs=list(outputs),
+        cancel_requested=bool(row["cancel_requested"]),
+        checkpoint=checkpoint,
+        request=request,
+    )
+
+
+class SqliteJobStore(JobStore):
+    """Durable job store. Jobs survive process restart."""
+
+    def __init__(self, path: str | Path, *, max_jobs: int = 200) -> None:
+        self.path = str(path)
+        # ":memory:" is allowed and useful for tests.
+        if self.path != ":memory:":
+            Path(self.path).expanduser().parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.RLock()
+        self._conn = sqlite3.connect(self.path, check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA foreign_keys=ON")
+        with self._lock:
+            self._conn.executescript(_SCHEMA)
+            self._migrate_locked()
+            self._conn.commit()
+        self._max_jobs = max_jobs
+
+    # -- interface ---------------------------------------------------------
+
+    def create(self, source: str, *, request: dict[str, Any] | None = None) -> Job:
+        now = time.time()
+        job = Job(
+            id=uuid.uuid4().hex[:12],
+            source=source,
+            state=JobState.PENDING,
+            created_at=now,
+            updated_at=now,
+            request=request,
+        )
+        with self._lock:
+            self._conn.execute(
+                "INSERT INTO jobs (id, source, state, created_at, updated_at,"
+                " progress, error, transcript, outputs, cancel_requested, checkpoint, request)"
+                " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (
+                    job.id,
+                    job.source,
+                    job.state.value,
+                    job.created_at,
+                    job.updated_at,
+                    job.progress,
+                    job.error,
+                    None,
+                    json.dumps([]),
+                    int(job.cancel_requested),
+                    None,
+                    json.dumps(request) if request is not None else None,
+                ),
+            )
+            self._conn.commit()
+            self._prune_locked()
+        return job
+
+    def get(self, job_id: str) -> Job | None:
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT * FROM jobs WHERE id = ?", (job_id,)
+            ).fetchone()
+        return _row_to_job(row) if row else None
+
+    def update(self, job_id: str, **fields: Any) -> Job | None:
+        patch = {k: v for k, v in fields.items() if k in _MUTABLE}
+        if not patch:
+            return self.get(job_id)
+
+        sets: list[str] = []
+        values: list[Any] = []
+        for key, value in patch.items():
+            sets.append(f"{key} = ?")
+            if key == "state" and isinstance(value, JobState):
+                values.append(value.value)
+            elif key == "transcript":
+                values.append(json.dumps(value) if value is not None else None)
+            elif key == "outputs":
+                values.append(json.dumps(list(value or [])))
+            elif key in {"checkpoint", "request"}:
+                values.append(json.dumps(value) if value is not None else None)
+            elif key == "cancel_requested":
+                values.append(int(bool(value)))
+            else:
+                values.append(value)
+
+        sets.append("updated_at = ?")
+        values.append(time.time())
+        values.append(job_id)
+
+        with self._lock:
+            cur = self._conn.execute(
+                f"UPDATE jobs SET {', '.join(sets)} WHERE id = ?",
+                values,
+            )
+            self._conn.commit()
+        if cur.rowcount == 0:
+            return None
+        return self.get(job_id)
+
+    def list(self, *, limit: int = 50, state: JobState | None = None) -> list[Job]:
+        if limit < 0:
+            raise ValueError("limit must be >= 0")
+        with self._lock:
+            if state is None:
+                rows = self._conn.execute(
+                    "SELECT * FROM jobs ORDER BY seq DESC LIMIT ?", (limit,)
+                ).fetchall()
+            else:
+                rows = self._conn.execute(
+                    "SELECT * FROM jobs WHERE state = ? ORDER BY seq DESC LIMIT ?",
+                    (state.value, limit),
+                ).fetchall()
+        return [_row_to_job(r) for r in rows]
+
+    def clear(self) -> None:
+        with self._lock:
+            self._conn.execute("DELETE FROM jobs")
+            self._conn.commit()
+
+    # -- internals ---------------------------------------------------------
+
+    def _migrate_locked(self) -> None:
+        """Add columns introduced after the original schema."""
+        existing = {
+            row["name"] for row in self._conn.execute("PRAGMA table_info(jobs)").fetchall()
+        }
+        if _CHECKPOINT_COLUMN not in existing:
+            self._conn.execute("ALTER TABLE jobs ADD COLUMN checkpoint TEXT")
+        if "request" not in existing:
+            self._conn.execute("ALTER TABLE jobs ADD COLUMN request TEXT")
+
+    def _prune_locked(self) -> None:
+        """Drop the oldest terminal jobs once over capacity."""
+        rows = self._conn.execute(
+            "SELECT seq FROM jobs WHERE state IN (?, ?, ?) ORDER BY seq ASC",
+            (JobState.DONE.value, JobState.ERROR.value, JobState.CANCELLED.value),
+        ).fetchall()
+        excess = self._count_locked() - self._max_jobs
+        if excess <= 0 or not rows:
+            return
+        doomed = [r["seq"] for r in rows[:excess]]
+        if doomed:
+            self._conn.executemany("DELETE FROM jobs WHERE seq = ?", [(s,) for s in doomed])
+            self._conn.commit()
+
+    def _count_locked(self) -> int:
+        return int(self._conn.execute("SELECT COUNT(*) AS n FROM jobs").fetchone()["n"])
+
+    def close(self) -> None:
+        with self._lock:
+            self._conn.close()
+

textflowkit/core/submission.py

@@ -0,0 +1,220 @@
+"""One submission contract for CLI, MCP, HTTP, and batch callers."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from textflowkit.core.checkpoint import (
+    find_resumable_checkpoint,
+    is_local_source,
+    load_checkpoint,
+    matches,
+    prepare_resume,
+    reusable_done_result,
+    validate_local_resume,
+)
+from textflowkit.core.executor import QueueFullError, get_default_executor
+from textflowkit.core.jobs import Job, JobState, JobStore
+from textflowkit.core.paths import default_input_root
+from textflowkit.core.runner import run_job
+from textflowkit.render import SUPPORTED_FORMATS
+
+
+@dataclass(slots=True)
+class SubmissionRequest:
+    source: str
+    language: str | None = None
+    formats: list[str] = field(default_factory=lambda: ["json", "srt", "txt"])
+    output_dir: str | None = None
+    model: str = "small"
+    engine: str = "whisper"
+    device: str | None = None
+    cookies_from_browser: str | None = None
+    input_root: str | None = None
+    work_dir: str | None = None
+    diarize: bool = False
+    diarizer_backend: str = "pyannote"
+    translate_to: str | None = None
+    translator_backend: str = "ollama"
+
+    def __post_init__(self) -> None:
+        if not self.source:
+            raise ValueError("source is required")
+        # Adapter path helpers return Path objects, but a durable request must
+        # be JSON-serializable before it is inserted into SQLite.
+        if isinstance(self.input_root, Path):
+            self.input_root = str(self.input_root)
+        if isinstance(self.work_dir, Path):
+            self.work_dir = str(self.work_dir)
+        self.formats = [str(fmt).lower().lstrip(".") for fmt in self.formats]
+        bad = [fmt for fmt in self.formats if fmt not in SUPPORTED_FORMATS]
+        if bad:
+            raise ValueError(f"unsupported format(s): {', '.join(bad)}")
+        if len(self.formats) != len(set(self.formats)):
+            raise ValueError("duplicate output format")
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> SubmissionRequest:
+        return cls(**data)
+
+    def options(self) -> dict[str, Any]:
+        return {
+            "formats": list(self.formats),
+            "diarize": self.diarize,
+            "diarizer_backend": self.diarizer_backend,
+            "translate_to": self.translate_to,
+            "translator_backend": self.translator_backend,
+        }
+
+    def run_kwargs(self) -> dict[str, Any]:
+        return self.to_dict()
+
+
+def _matching_checkpoint(store: JobStore, request: SubmissionRequest, job_id: str | None):
+    if job_id is not None:
+        job = store.get(job_id)
+        if job is None:
+            raise ValueError(f"no job with id '{job_id}'")
+        if job.request is not None:
+            saved = dict(job.request)
+            incoming = request.to_dict()
+            for key in ("input_root", "work_dir"):
+                saved.pop(key, None)
+                incoming.pop(key, None)
+            if saved != incoming:
+                raise ValueError("resume request does not match the saved request")
+        checkpoint = load_checkpoint(job)
+        if checkpoint is None:
+            return job, None
+        if not matches(
+            checkpoint, source=request.source, model=request.model,
+            language=request.language, engine=request.engine, device=request.device,
+            options=request.options(),
+        ):
+            raise ValueError("resume request does not match the saved checkpoint")
+        return job, checkpoint
+    return find_resumable_checkpoint(
+        store, source=request.source, model=request.model,
+        language=request.language, engine=request.engine, device=request.device,
+        options=request.options(),
+    )
+
+
+def submit_request(
+    store: JobStore,
+    request: SubmissionRequest,
+    *,
+    background: bool = True,
+    resume: bool = False,
+    resume_job_id: str | None = None,
+) -> Job:
+    """Submit or resume via the same durable job lifecycle on every surface."""
+    executor = get_default_executor() if background else None
+    if executor is not None and executor.store is store:
+        # Reap old PENDING/RUNNING records before selecting one to resume.
+        executor.start()
+    found = _matching_checkpoint(store, request, resume_job_id) if resume or resume_job_id else None
+    if found is not None:
+        prior, checkpoint = found
+        if is_local_source(request.source):
+            if checkpoint is None:
+                if prior.state is JobState.DONE:
+                    raise ValueError("local job has no reusable checkpoint; resubmit without resume")
+            else:
+                validate_local_resume(
+                    checkpoint, request.source,
+                    input_root=request.input_root if request.input_root is not None
+                    else default_input_root(),
+                )
+        if prior.state is JobState.DONE:
+            if request.output_dir is None:
+                return prior
+            result = reusable_done_result(store, prior, formats=request.formats,
+                                          output_dir=request.output_dir,
+                                          stem=_output_stem(prior))
+            if result is not None:
+                _transcript, outputs = result
+                updated = store.update(prior.id, outputs=[str(p) for p in outputs])
+                return updated or prior
+        if prior.state in {JobState.PENDING, JobState.RUNNING}:
+            raise ValueError(f"job '{prior.id}' is already active")
+        prepared = prepare_resume(store, prior, checkpoint) if checkpoint else None
+        if checkpoint is None:
+            reopened = store.update(
+                prior.id, state=JobState.PENDING, progress="resuming from start",
+                error=None, cancel_requested=False,
+            )
+            if reopened is None:
+                raise ValueError(f"job '{prior.id}' disappeared before resume")
+            prepared = reopened, None
+        if prepared is not None:
+            job, checkpoint_payload = prepared
+            kwargs = request.run_kwargs()
+            store.update(job.id, request=request.to_dict())
+            if checkpoint_payload is not None:
+                kwargs["resume_checkpoint"] = checkpoint_payload
+            if executor is not None and executor.store is store:
+                try:
+                    return executor.enqueue(job, **kwargs)
+                except QueueFullError:
+                    store.update(job.id, state=JobState.ERROR, progress="queue full",
+                                 error="resume not queued; job queue is full")
+                    raise
+            run_job(job, store, **kwargs)
+            return store.get(job.id) or job
+
+    kwargs = request.run_kwargs()
+    if executor is not None and executor.store is store:
+        return executor.submit(request=request.to_dict(), **kwargs)
+    job = store.create(request.source, request=request.to_dict())
+    run_job(job, store, **kwargs)
+    return store.get(job.id) or job
+
+
+def resume_job(
+    store: JobStore, job_id: str, *, background: bool = True,
+    input_root: str | None = None, work_dir: str | None = None,
+) -> Job:
+    """Resume an interrupted durable job using its original saved request."""
+    job = store.get(job_id)
+    if job is None:
+        raise ValueError(f"no job with id '{job_id}'")
+    if job.request is None:
+        raise ValueError("job predates saved requests; resubmit its original options")
+    request_data = dict(job.request)
+    if input_root is not None:
+        request_data["input_root"] = input_root
+    if work_dir is not None:
+        request_data["work_dir"] = work_dir
+    return submit_request(
+        store, SubmissionRequest.from_dict(request_data),
+        background=background, resume=True, resume_job_id=job_id,
+    )
+
+
+def submit_batch(
+    store: JobStore, requests: list[SubmissionRequest], *, resume: bool = False
+) -> list[dict[str, Any]]:
+    """Accept each source independently; a bad item never hides later items."""
+    results: list[dict[str, Any]] = []
+    for request in requests:
+        try:
+            job = submit_request(store, request, resume=resume)
+            results.append({"source": request.source, "job_id": job.id, "state": job.state.value})
+        except Exception as exc:  # noqa: BLE001 - one bad item must not stop the batch
+            results.append({"source": request.source, "error": f"{type(exc).__name__}: {exc}"})
+    return results
+
+
+def _output_stem(job: Job) -> str:
+    if job.outputs:
+        return Path(job.outputs[0]).stem
+    parsed = urlparse(job.source)
+    name = Path(parsed.path).stem if parsed.scheme in {"http", "https"} else Path(job.source).stem
+    return f"{name or 'transcript'}-{job.id}"

textflowkit/core/timeutil.py

@@ -0,0 +1,20 @@
+"""Timestamp formatting for subtitle formats."""
+
+from __future__ import annotations
+
+
+def srt_timestamp(seconds: float) -> str:
+    """HH:MM:SS,mmm (SRT uses a comma)."""
+    if seconds < 0:
+        seconds = 0.0
+    ms_total = round(seconds * 1000)
+    h, rem = divmod(ms_total, 3_600_000)
+    m, rem = divmod(rem, 60_000)
+    s, ms = divmod(rem, 1000)
+    return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
+
+
+def vtt_timestamp(seconds: float) -> str:
+    """HH:MM:SS.mmm (WebVTT uses a period)."""
+    return srt_timestamp(seconds).replace(",", ".")
+