textflowkit 0.1.3__py3-none-any.whl

textflowkit/core/jobs.py

@@ -0,0 +1,241 @@
+"""Job model and the store interface.
+
+Long-running transcription is modelled as a job so the same interface works
+everywhere: stdio MCP polls in-process, an HTTP adapter polls over the wire, and
+a website can queue work.
+
+`JobStore` is the abstraction. Two implementations ship:
+
+- `MemoryJobStore` - fast, process-local, used for tests and ephemeral runs.
+- `SqliteJobStore` - durable, survives restart. Selected by setting
+  ``TEXTFLOWKIT_DB``.
+
+Ordering is always by insertion sequence, never by wall-clock time: on Windows
+with Python < 3.13 ``time.time()`` is coarse enough that jobs created in a tight
+loop share a timestamp, and tie-breaking by insertion order silently inverts
+"newest first".
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import time
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class JobState(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    DONE = "done"
+    ERROR = "error"
+    CANCELLED = "cancelled"
+
+
+TERMINAL_STATES = frozenset({JobState.DONE, JobState.ERROR, JobState.CANCELLED})
+MAX_LIST_LIMIT = 1000
+
+
+def validate_list_limit(limit: int) -> int:
+    if limit < 0 or limit > MAX_LIST_LIMIT:
+        raise ValueError(f"limit must be between 0 and {MAX_LIST_LIMIT}")
+    return limit
+
+# A job in one of these states expects a worker to be running it. Across a
+# restart there is no worker, so any such job is orphaned and must be reaped -
+# otherwise it sits in "running" forever.
+INCOMPLETE_STATES = frozenset({JobState.PENDING, JobState.RUNNING})
+
+
+@dataclass
+class Job:
+    """A single transcription job."""
+
+    id: str
+    source: str
+    state: JobState = JobState.PENDING
+    created_at: float = field(default_factory=time.time)
+    updated_at: float = field(default_factory=time.time)
+    progress: str = ""
+    error: str | None = None
+    transcript: dict[str, Any] | None = None
+    outputs: list[str] = field(default_factory=list)
+    cancel_requested: bool = False
+    checkpoint: dict[str, Any] | None = None
+    request: dict[str, Any] | None = None
+
+    @property
+    def is_terminal(self) -> bool:
+        return self.state in TERMINAL_STATES
+
+    def to_dict(
+        self, *, include_transcript: bool = False, include_checkpoint: bool = False
+    ) -> dict[str, Any]:
+        data = {
+            "id": self.id,
+            "source": self.source,
+            "state": self.state.value,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "progress": self.progress,
+            "error": self.error,
+            "outputs": list(self.outputs),
+            "cancel_requested": self.cancel_requested,
+        }
+        if include_checkpoint and self.checkpoint is not None:
+            data["checkpoint"] = self.checkpoint
+        if include_transcript and self.transcript is not None:
+            data["transcript"] = self.transcript
+        return data
+
+
+class JobStore(ABC):
+    """Storage for jobs.
+
+    Implementations must be safe for concurrent use from multiple threads, and
+    must return jobs ordered newest-first by insertion sequence.
+    """
+
+    @abstractmethod
+    def create(self, source: str, *, request: dict[str, Any] | None = None) -> Job:
+        """Create a pending job and return it."""
+
+    @abstractmethod
+    def get(self, job_id: str) -> Job | None:
+        """Fetch one job, or None."""
+
+    @abstractmethod
+    def update(self, job_id: str, **fields: Any) -> Job | None:
+        """Patch fields on a job and bump updated_at. None if absent."""
+
+    @abstractmethod
+    def list(self, *, limit: int = 50, state: JobState | None = None) -> list[Job]:
+        """Recent jobs, newest first, optionally filtered by state."""
+
+    @abstractmethod
+    def clear(self) -> None:
+        """Remove every job."""
+
+    def reap_incomplete(self, *, reason: str) -> int:
+        """Fail any job left mid-flight, returning how many were reaped.
+
+        Called at startup. A job in PENDING/RUNNING has no worker after a
+        restart, so leaving it alone would report a job that will never finish.
+        """
+        reaped = 0
+        for job in self.list(limit=10_000):
+            if job.state in INCOMPLETE_STATES:
+                self.update(
+                    job.id,
+                    state=JobState.ERROR,
+                    error=reason,
+                    progress="interrupted",
+                )
+                reaped += 1
+        return reaped
+
+    def close(self) -> None:
+        """Release resources. No-op by default."""
+
+
+class MemoryJobStore(JobStore):
+    """In-memory, thread-safe job store. Fast; not durable."""
+
+    def __init__(self, *, max_jobs: int = 200) -> None:
+        self._jobs: dict[str, Job] = {}
+        self._order: list[str] = []
+        self._lock = threading.RLock()
+        self._max_jobs = max_jobs
+
+    def create(self, source: str, *, request: dict[str, Any] | None = None) -> Job:
+        job = Job(id=uuid.uuid4().hex[:12], source=source, request=request)
+        with self._lock:
+            self._jobs[job.id] = job
+            self._order.append(job.id)
+            self._evict_locked()
+        return job
+
+    def get(self, job_id: str) -> Job | None:
+        with self._lock:
+            return self._jobs.get(job_id)
+
+    def update(self, job_id: str, **fields: Any) -> Job | None:
+        with self._lock:
+            job = self._jobs.get(job_id)
+            if job is None:
+                return None
+            for key, value in fields.items():
+                setattr(job, key, value)
+            job.updated_at = time.time()
+            return job
+
+    def list(self, *, limit: int = 50, state: JobState | None = None) -> list[Job]:
+        if limit < 0:
+            raise ValueError("limit must be >= 0")
+        with self._lock:
+            jobs = [self._jobs[i] for i in reversed(self._order) if i in self._jobs]
+        if state is not None:
+            jobs = [j for j in jobs if j.state == state]
+        return jobs[:limit]
+
+    def _evict_locked(self) -> None:
+        """Drop oldest terminal jobs once over capacity."""
+        while len(self._order) > self._max_jobs:
+            for idx, job_id in enumerate(self._order):
+                job = self._jobs.get(job_id)
+                if job is None:
+                    self._order.pop(idx)
+                    break
+                if job.is_terminal:
+                    self._order.pop(idx)
+                    self._jobs.pop(job_id, None)
+                    break
+            else:
+                return  # everything still running; keep them all
+
+    def clear(self) -> None:
+        with self._lock:
+            self._jobs.clear()
+            self._order.clear()
+
+
+ENV_DB = "TEXTFLOWKIT_DB"
+
+_default_store: JobStore | None = None
+_store_lock = threading.Lock()
+
+
+def _make_store() -> JobStore:
+    """Choose a store from the environment. Durable when TEXTFLOWKIT_DB is set."""
+    db_path = os.environ.get(ENV_DB)
+    if db_path:
+        from textflowkit.core.sqlite_store import SqliteJobStore
+
+        return SqliteJobStore(db_path)
+    return MemoryJobStore()
+
+
+def get_default_store() -> JobStore:
+    """Process-wide store shared by the MCP and HTTP adapters."""
+    global _default_store
+    with _store_lock:
+        if _default_store is None:
+            _default_store = _make_store()
+        return _default_store
+
+
+def set_default_store(store: JobStore | None) -> None:
+    """Override the process-wide store (tests, embedding)."""
+    global _default_store
+    with _store_lock:
+        _default_store = store
+
+
+def reset_default_store() -> None:
+    """Drop the cached store so the next call re-reads the environment."""
+    set_default_store(None)
+

textflowkit/core/model.py

@@ -0,0 +1,98 @@
+"""Canonical transcript data model.
+
+Every source, engine, and renderer speaks this shape. Keeping one canonical
+object is what lets the pipeline stay shared while platforms multiply.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(slots=True)
+class Segment:
+    """One timestamped span of speech."""
+
+    start: float
+    end: float
+    text: str
+    speaker: str | None = None
+    translated_text: str | None = None
+    hidden: bool = False
+
+    def display_text(self) -> str:
+        """Text to render: translated if present, else source."""
+        return self.translated_text or self.text
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Segment:
+        return cls(
+            start=float(data["start"]),
+            end=float(data["end"]),
+            text=str(data["text"]),
+            speaker=data.get("speaker"),
+            translated_text=data.get("translated_text"),
+            hidden=bool(data.get("hidden", False)),
+        )
+
+
+@dataclass(slots=True)
+class Transcript:
+    """A complete transcription result."""
+
+    source: str
+    language: str | None = None
+    segments: list[Segment] = field(default_factory=list)
+    platform: str | None = None
+    duration: float | None = None
+    engine: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def text(self) -> str:
+        return "\n".join(s.display_text().strip() for s in self.segments if not s.hidden)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "language": self.language,
+            "platform": self.platform,
+            "duration": self.duration,
+            "engine": self.engine,
+            "metadata": self.metadata,
+            "segments": [s.to_dict() for s in self.segments],
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Transcript:
+        return cls(
+            source=data.get("source", ""),
+            language=data.get("language"),
+            platform=data.get("platform"),
+            duration=data.get("duration"),
+            engine=data.get("engine"),
+            metadata=data.get("metadata", {}) or {},
+            segments=[Segment.from_dict(s) for s in data.get("segments", [])],
+        )
+
+    def to_json(self, *, indent: int = 2) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=indent)
+
+    @classmethod
+    def from_json(cls, raw: str) -> Transcript:
+        return cls.from_dict(json.loads(raw))
+
+    def save_json(self, path: str | Path) -> Path:
+        p = Path(path)
+        p.write_text(self.to_json(), encoding="utf-8")
+        return p
+
+    @classmethod
+    def load_json(cls, path: str | Path) -> Transcript:
+        return cls.from_json(Path(path).read_text(encoding="utf-8"))

textflowkit/core/paths.py

@@ -0,0 +1,226 @@
+"""Output path resolution.
+
+Adapters accept a destination directory from a caller - a CLI user, an HTTP
+client, or a model. That value must not be able to name an arbitrary location on
+the host.
+
+Policy:
+
+- The process chooses an allowed root. `TEXTFLOWKIT_OUTPUT_ROOT` sets it; when
+  unset the root is the current working directory. This keeps the CLI's default
+  ("write where I ran it") working while giving a server operator one switch to
+  confine every write.
+- A requested directory must resolve *inside* that root. `..` segments, absolute
+  paths elsewhere, and symlinks that escape are all rejected.
+- The returned path is the resolved absolute path, so callers never re-interpret
+  the raw string.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+ENV_OUTPUT_ROOT = "TEXTFLOWKIT_OUTPUT_ROOT"
+ENV_INPUT_ROOT = "TEXTFLOWKIT_INPUT_ROOT"
+
+
+class UnsafeOutputPathError(ValueError):
+    """Raised when a requested output directory escapes the allowed root."""
+
+
+class UnsafeInputPathError(ValueError):
+    """Raised when a local input path escapes the allowed input root."""
+
+
+def output_root() -> Path:
+    """The directory all rendered output must live under.
+
+    Default: the current working directory, which is the least surprising answer
+    for a CLI ("write where I ran it"). A caller that asks for somewhere else
+    inside the same machine is not doing anything the operator could not, so
+    `TEXTFLOWKIT_OUTPUT_ROOT` is the switch that imposes a real boundary when a
+    deployment needs one.
+    """
+    raw = os.environ.get(ENV_OUTPUT_ROOT)
+    base = Path(raw) if raw else Path.cwd()
+    return base.expanduser().resolve()
+
+
+def output_is_confined() -> bool:
+    """Whether an output boundary was explicitly requested.
+
+    Distinguishes "the operator chose a root" from "we fell back to cwd". The
+    CLI writes wherever asked when no root is set; an explicitly configured root
+    is still enforced exactly as before.
+    """
+    return bool(os.environ.get(ENV_OUTPUT_ROOT))
+
+
+def resolve_output_dir(requested: str | None) -> Path:
+    """Resolve a caller-supplied output directory.
+
+    Confinement applies only when `TEXTFLOWKIT_OUTPUT_ROOT` is set. Without it
+    there is no boundary to violate: the caller is the operator and already has
+    whatever access the machine gives them, so refusing a path they own would be
+    the tool inventing a restriction rather than enforcing one.
+
+    Raises `UnsafeOutputPathError` when an explicit root is set and the path
+    escapes it.
+    """
+    root = output_root()
+
+    if requested is None or requested == "":
+        target = root
+    else:
+        candidate = Path(requested).expanduser()
+        target = candidate if candidate.is_absolute() else (root / candidate)
+
+    # strict=False: the directory may not exist yet.
+    resolved = target.resolve()
+
+    if output_is_confined() and resolved != root and root not in resolved.parents:
+        raise UnsafeOutputPathError(
+            f"output directory '{requested}' is outside the allowed root "
+            f"'{root}'. Set {ENV_OUTPUT_ROOT} to widen the root, or choose a "
+            "path inside it."
+        )
+    return resolved
+
+
+def ensure_output_dir(requested: str | None) -> Path:
+    """Resolve and create the output directory."""
+    resolved = resolve_output_dir(requested)
+    resolved.mkdir(parents=True, exist_ok=True)
+    # Recheck after creation: a symlink/junction may have changed since the
+    # first resolve. File publishing repeats the check at point of use.
+    return resolve_output_dir(str(resolved))
+
+
+def verify_output_file_target(path: Path) -> None:
+    """Check the destination parent immediately before publishing a file."""
+    if not output_is_confined():
+        return
+    root = output_root()
+    parent = path.parent.resolve(strict=True)
+    if parent != root and root not in parent.parents:
+        raise UnsafeOutputPathError(
+            f"output file '{path}' is outside the allowed root '{root}'"
+        )
+
+
+# --- input confinement ----------------------------------------------------
+
+def resolve_input_path(requested: str | Path, *, root: str | Path | None) -> Path:
+    """Resolve a local input path, optionally confined to `root`.
+
+    Confinement is opt-in. When `root` is None the path is returned resolved but
+    unrestricted, because the caller is the principal - a person who typed the
+    path, or an agent acting with their authority. Adapters also pass None by
+    default; set TEXTFLOWKIT_INPUT_ROOT when the caller is *not* the machine's
+    owner (a shared or network-reachable deployment).
+
+    Raises `UnsafeInputPathError` when the path is outside the root, or when it
+    does not name readable regular file.
+    """
+    candidate = Path(requested).expanduser()
+
+    if root is None:
+        resolved = candidate.resolve()
+        if not resolved.exists():
+            raise FileNotFoundError(f"no such file: {requested}")
+        if not resolved.is_file():
+            raise ValueError(f"not a file: {requested}")
+        return resolved
+
+    base = Path(root).expanduser().resolve()
+    target = candidate if candidate.is_absolute() else (base / candidate)
+    resolved = target.resolve()
+
+    if resolved != base and base not in resolved.parents:
+        raise UnsafeInputPathError(
+            f"input path '{requested}' is outside the allowed input root "
+            f"'{base}'. Set {ENV_INPUT_ROOT} to widen the root, or use a path "
+            "inside it."
+        )
+    if not resolved.exists():
+        raise FileNotFoundError(f"no such file: {requested}")
+    if not resolved.is_file():
+        raise ValueError(f"not a file: {requested}")
+    return resolved
+
+
+def opened_file_path(fd: int, fallback: Path) -> Path:
+    """Resolve the path of the *opened handle*, not a name checked earlier."""
+    if os.name == "nt":
+        import ctypes
+        import msvcrt
+        from ctypes import wintypes
+
+        kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+        fn = kernel.GetFinalPathNameByHandleW
+        fn.argtypes = [wintypes.HANDLE, wintypes.LPWSTR, wintypes.DWORD, wintypes.DWORD]
+        fn.restype = wintypes.DWORD
+        buffer = ctypes.create_unicode_buffer(32768)
+        length = fn(msvcrt.get_osfhandle(fd), buffer, len(buffer), 0)
+        if not length or length >= len(buffer):
+            raise UnsafeInputPathError("could not verify the opened input file path")
+        raw = buffer.value
+        if raw.startswith("\\\\?\\UNC\\"):
+            raw = "\\\\" + raw[8:]
+        elif raw.startswith("\\\\?\\"):
+            raw = raw[4:]
+        return Path(raw).resolve()
+    if os.path.exists(f"/proc/self/fd/{fd}"):
+        return Path(os.readlink(f"/proc/self/fd/{fd}")).resolve()
+    if sys.platform == "darwin":
+        try:
+            return _darwin_opened_file_path(fd)
+        except (OSError, ValueError) as exc:
+            raise UnsafeInputPathError(
+                f"cannot verify opened input file path: {fallback}: {exc}"
+            ) from exc
+    raise UnsafeInputPathError(f"cannot verify opened input file path: {fallback}")
+
+
+def _darwin_opened_file_path(fd: int) -> Path:
+    """Use Apple's F_GETPATH even when Python omits the symbolic constant."""
+    import fcntl
+
+    # Apple bsd/sys/fcntl.h defines F_GETPATH as 50. MAXPATHLEN is 1024;
+    # Python 3.10-3.13 fcntl() also caps the argument buffer at 1024 bytes.
+    # A larger buffer raises ValueError before the OS call.
+    command = getattr(fcntl, "F_GETPATH", 50)
+    raw = fcntl.fcntl(fd, command, b"\0" * 1024)
+    path = os.fsdecode(raw.split(b"\0", 1)[0])
+    if not path:
+        raise ValueError("F_GETPATH returned an empty path")
+    return Path(path).resolve()
+
+
+def default_input_root() -> Path | None:
+    """The configured input root, or None when confinement is not requested."""
+    raw = os.environ.get(ENV_INPUT_ROOT)
+    return Path(raw).expanduser().resolve() if raw else None
+
+
+def server_input_root() -> Path | None:
+    """The input root a long-running adapter should enforce, or None for no limit.
+
+    Default: **no confinement.** An adapter runs as the person who started it and
+    inherits their access to the machine. When that person is the operator - a
+    developer running this for themselves, or an agent acting on their behalf -
+    confining it to the working directory only refuses paths they are already
+    entitled to use, which reads as the tool being broken.
+
+    Confinement is still one environment variable away for the case it was
+    actually designed for: exposing a server to callers who are *not* the owner.
+    A shared or network-reachable deployment should set:
+
+        TEXTFLOWKIT_INPUT_ROOT=/srv/media   # one directory
+        TEXTFLOWKIT_INPUT_ROOT=C:\\media      # one tree on Windows
+
+    Setting it to a filesystem root is equivalent to no confinement.
+    """
+    return default_input_root()