fow-cli 0.1.0__py3-none-any.whl

fly_on_the_wall/meetings.py

@@ -0,0 +1,382 @@
+from __future__ import annotations
+
+import hashlib
+import re
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from sqlite3 import Connection
+from uuid import uuid4
+
+from fly_on_the_wall.audio_metadata import extract_and_store_audio_metadata
+from fly_on_the_wall.config import AppConfig
+from fly_on_the_wall.storage import StoragePaths, ensure_storage_layout
+
+
+@dataclass(frozen=True)
+class Meeting:
+    id: str
+    slug: str
+    title: str
+    title_source: str
+    language: str
+    imported_audio_path: Path
+    audio_sha256: str | None = None
+    generated_title: str | None = None
+
+
+@dataclass(frozen=True)
+class DeleteMeetingResult:
+    id: str
+    slug: str
+    removed_paths: tuple[Path, ...]
+
+
+def import_meeting(
+    connection: Connection,
+    audio_path: Path,
+    title: str | None,
+    config: AppConfig,
+    storage: StoragePaths | None = None,
+    description: str | None = None,
+) -> Meeting:
+    if not audio_path.is_file():
+        raise FileNotFoundError(f"Audio file does not exist: {audio_path}")
+
+    audio_sha256 = file_sha256(audio_path)
+    existing = get_meeting_by_audio_sha256(connection, audio_sha256)
+    if existing is not None:
+        return _meeting_from_row(existing)
+
+    paths = storage or ensure_storage_layout()
+    meeting_id = str(uuid4())
+    provisional_title = title or audio_path.stem
+    title_source = "manual" if title else "filename"
+    slug = unique_slug(connection, slugify(provisional_title))
+    imported_audio_path = paths.audio / slug / audio_path.name
+    imported_audio_path.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(audio_path, imported_audio_path)
+
+    with connection:
+        connection.execute(
+            """
+            INSERT INTO meetings(
+                id,
+                slug,
+                title,
+                title_source,
+                description,
+                language,
+                original_audio_path,
+                imported_audio_path,
+                audio_sha256
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            (
+                meeting_id,
+                slug,
+                provisional_title,
+                title_source,
+                description,
+                config.language,
+                str(audio_path),
+                str(imported_audio_path),
+                audio_sha256,
+            ),
+        )
+        extract_and_store_audio_metadata(connection, meeting_id, imported_audio_path, paths)
+
+    return Meeting(
+        id=meeting_id,
+        slug=slug,
+        title=provisional_title,
+        title_source=title_source,
+        language=config.language,
+        imported_audio_path=imported_audio_path,
+        audio_sha256=audio_sha256,
+    )
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def slugify(value: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return slug or "meeting"
+
+
+def unique_slug(connection: Connection, base_slug: str) -> str:
+    slug = base_slug
+    suffix = 2
+    while _slug_exists(connection, slug):
+        slug = f"{base_slug}-{suffix}"
+        suffix += 1
+    return slug
+
+
+def _slug_exists(connection: Connection, slug: str) -> bool:
+    row = connection.execute("SELECT 1 FROM meetings WHERE slug = ?", (slug,)).fetchone()
+    return row is not None
+
+
+def list_meetings(connection: Connection) -> list[dict]:
+    return [
+        dict(row)
+        for row in connection.execute(
+            """
+            SELECT id, slug, title, title_source, generated_title, language, created_at
+            FROM meetings
+            ORDER BY created_at DESC
+            """
+        ).fetchall()
+    ]
+
+
+def get_meeting(connection: Connection, meeting_id_or_slug: str) -> dict | None:
+    row = connection.execute(
+        """
+        SELECT * FROM meetings
+        WHERE id = ? OR slug = ?
+        """,
+        (meeting_id_or_slug, meeting_id_or_slug),
+    ).fetchone()
+    return None if row is None else dict(row)
+
+
+def get_meeting_by_audio_sha256(connection: Connection, audio_sha256: str) -> dict | None:
+    row = connection.execute("SELECT * FROM meetings WHERE audio_sha256 = ?", (audio_sha256,)).fetchone()
+    return None if row is None else dict(row)
+
+
+def latest_completed_provider_run(connection: Connection, meeting_id: str, provider: str = "elevenlabs") -> dict | None:
+    row = connection.execute(
+        """
+        SELECT * FROM provider_runs
+        WHERE meeting_id = ? AND provider = ? AND status = 'done'
+        ORDER BY completed_at DESC, created_at DESC
+        LIMIT 1
+        """,
+        (meeting_id, provider),
+    ).fetchone()
+    return None if row is None else dict(row)
+
+
+def update_generated_title(connection: Connection, meeting_id: str, generated_title: str) -> None:
+    normalized_title = generated_title.strip()
+    if not normalized_title:
+        return
+
+    with connection:
+        row = connection.execute("SELECT title_source FROM meetings WHERE id = ?", (meeting_id,)).fetchone()
+        if row is None:
+            raise ValueError(f"Meeting not found: {meeting_id}")
+
+        if row["title_source"] == "manual":
+            connection.execute(
+                """
+                UPDATE meetings
+                SET generated_title = ?, updated_at = CURRENT_TIMESTAMP
+                WHERE id = ?
+                """,
+                (normalized_title, meeting_id),
+            )
+        else:
+            connection.execute(
+                """
+                UPDATE meetings
+                SET title = ?, title_source = 'generated', generated_title = ?,
+                    updated_at = CURRENT_TIMESTAMP
+                WHERE id = ?
+                """,
+                (normalized_title, normalized_title, meeting_id),
+            )
+
+
+def rename_meeting(connection: Connection, meeting_id_or_slug: str, title: str) -> dict:
+    meeting = get_meeting(connection, meeting_id_or_slug)
+    if meeting is None:
+        raise ValueError(f"Meeting not found: {meeting_id_or_slug}")
+
+    normalized_title = title.strip()
+    if not normalized_title:
+        raise ValueError("Meeting title cannot be empty.")
+
+    with connection:
+        connection.execute(
+            """
+            UPDATE meetings
+            SET title = ?, title_source = 'manual', updated_at = CURRENT_TIMESTAMP
+            WHERE id = ?
+            """,
+            (normalized_title, meeting["id"]),
+        )
+    updated = get_meeting(connection, meeting["id"])
+    if updated is None:
+        raise ValueError(f"Meeting not found: {meeting_id_or_slug}")
+    return updated
+
+
+def _meeting_from_row(row: dict) -> Meeting:
+    return Meeting(
+        id=row["id"],
+        slug=row["slug"],
+        title=row["title"],
+        title_source=row.get("title_source", "manual"),
+        language=row["language"],
+        imported_audio_path=Path(row["imported_audio_path"]),
+        audio_sha256=row.get("audio_sha256"),
+        generated_title=row.get("generated_title"),
+    )
+
+
+def meeting_stage_status(connection: Connection, meeting_id_or_slug: str) -> list[dict]:
+    meeting = get_meeting(connection, meeting_id_or_slug)
+    if meeting is None:
+        return []
+    return [
+        dict(row)
+        for row in connection.execute(
+            """
+            SELECT stage_name, status, error_message, updated_at
+            FROM pipeline_stages
+            WHERE meeting_id = ?
+            ORDER BY stage_name
+            """,
+            (meeting["id"],),
+        ).fetchall()
+    ]
+
+
+def delete_meeting(
+    connection: Connection,
+    meeting_id_or_slug: str,
+    storage: StoragePaths | None = None,
+    delete_published: bool = False,
+) -> DeleteMeetingResult:
+    meeting = get_meeting(connection, meeting_id_or_slug)
+    if meeting is None:
+        raise ValueError(f"Meeting not found: {meeting_id_or_slug}")
+
+    paths = storage or ensure_storage_layout()
+    removed_paths = _meeting_owned_paths(connection, meeting, paths)
+    if delete_published:
+        removed_paths.extend(_published_paths(connection, meeting["id"]))
+    local_speaker_ids = _local_speaker_ids(connection, meeting["id"])
+    voice_sample_ids = _meeting_voice_sample_ids(connection, meeting["id"])
+
+    with connection:
+        _delete_corrections(connection, meeting["id"], local_speaker_ids)
+        _delete_voice_samples(connection, voice_sample_ids)
+        connection.execute("DELETE FROM meetings WHERE id = ?", (meeting["id"],))
+
+    for path in removed_paths:
+        _remove_path(path)
+
+    return DeleteMeetingResult(
+        id=meeting["id"],
+        slug=meeting["slug"],
+        removed_paths=tuple(path for path in removed_paths if not path.exists()),
+    )
+
+
+def _meeting_owned_paths(connection: Connection, meeting: dict, storage: StoragePaths) -> list[Path]:
+    paths = [
+        storage.audio / meeting["slug"],
+        storage.artifacts / meeting["id"],
+        storage.exports / meeting["slug"],
+    ]
+
+    for key in ("imported_audio_path",):
+        if meeting.get(key):
+            paths.append(Path(meeting[key]))
+
+    for row in connection.execute(
+        "SELECT output_dir, manifest_path FROM exports WHERE meeting_id = ?", (meeting["id"],)
+    ).fetchall():
+        paths.append(Path(row["output_dir"]))
+        paths.append(Path(row["manifest_path"]))
+
+    for row in connection.execute(
+        """
+        SELECT audio_path, embedding_path
+        FROM voice_samples
+        WHERE source_meeting_id = ?
+        """,
+        (meeting["id"],),
+    ).fetchall():
+        paths.append(Path(row["audio_path"]))
+        if row["embedding_path"]:
+            paths.append(Path(row["embedding_path"]))
+
+    return _deduplicate_paths(paths)
+
+
+def _published_paths(connection: Connection, meeting_id: str) -> list[Path]:
+    return [
+        Path(row["output_path"])
+        for row in connection.execute(
+            "SELECT output_path FROM published_items WHERE meeting_id = ?",
+            (meeting_id,),
+        ).fetchall()
+    ]
+
+
+def _local_speaker_ids(connection: Connection, meeting_id: str) -> list[str]:
+    return [
+        row["id"]
+        for row in connection.execute("SELECT id FROM local_speakers WHERE meeting_id = ?", (meeting_id,)).fetchall()
+    ]
+
+
+def _meeting_voice_sample_ids(connection: Connection, meeting_id: str) -> list[str]:
+    return [
+        row["id"]
+        for row in connection.execute(
+            "SELECT id FROM voice_samples WHERE source_meeting_id = ?", (meeting_id,)
+        ).fetchall()
+    ]
+
+
+def _delete_corrections(connection: Connection, meeting_id: str, local_speaker_ids: list[str]) -> None:
+    connection.execute("DELETE FROM corrections WHERE meeting_id = ?", (meeting_id,))
+    if local_speaker_ids:
+        placeholders = ", ".join("?" for _ in local_speaker_ids)
+        connection.execute(
+            f"DELETE FROM corrections WHERE local_speaker_id IN ({placeholders})",
+            local_speaker_ids,
+        )
+
+
+def _delete_voice_samples(connection: Connection, voice_sample_ids: list[str]) -> None:
+    if not voice_sample_ids:
+        return
+    placeholders = ", ".join("?" for _ in voice_sample_ids)
+    connection.execute(
+        f"DELETE FROM voice_samples WHERE id IN ({placeholders})",
+        voice_sample_ids,
+    )
+
+
+def _remove_path(path: Path) -> None:
+    if not path.exists():
+        return
+    if path.is_dir():
+        shutil.rmtree(path)
+    else:
+        path.unlink()
+
+
+def _deduplicate_paths(paths: list[Path]) -> list[Path]:
+    seen: set[Path] = set()
+    deduplicated: list[Path] = []
+    for path in sorted(paths, key=lambda item: len(item.parts), reverse=True):
+        resolved = path.expanduser()
+        if resolved not in seen:
+            seen.add(resolved)
+            deduplicated.append(resolved)
+    return deduplicated

fly_on_the_wall/normalization.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from sqlite3 import Connection
+from typing import Any
+from uuid import uuid4
+
+
+@dataclass(frozen=True)
+class NormalizedSegment:
+    id: str
+    sequence: int
+    speaker_label: str
+    text: str
+    start_time: float | None
+    end_time: float | None
+    language: str | None
+
+
+def normalize_provider_run(connection: Connection, provider_run_id: str) -> list[NormalizedSegment]:
+    provider_run = connection.execute("SELECT * FROM provider_runs WHERE id = ?", (provider_run_id,)).fetchone()
+    if provider_run is None:
+        raise ValueError(f"Provider run does not exist: {provider_run_id}")
+
+    raw_response_path = Path(provider_run["raw_response_path"])
+    response = json.loads(raw_response_path.read_text())
+    segments = normalize_elevenlabs_response(response)
+
+    with connection:
+        connection.execute("DELETE FROM segments WHERE provider_run_id = ?", (provider_run_id,))
+        for segment in segments:
+            local_speaker_id = _ensure_local_speaker(
+                connection,
+                provider_run["meeting_id"],
+                provider_run_id,
+                segment.speaker_label,
+            )
+            connection.execute(
+                """
+                INSERT INTO segments(
+                    id,
+                    meeting_id,
+                    provider_run_id,
+                    local_speaker_id,
+                    sequence,
+                    start_time,
+                    end_time,
+                    text,
+                    language,
+                    source_json
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    segment.id,
+                    provider_run["meeting_id"],
+                    provider_run_id,
+                    local_speaker_id,
+                    segment.sequence,
+                    segment.start_time,
+                    segment.end_time,
+                    segment.text,
+                    segment.language,
+                    json.dumps({"speaker_label": segment.speaker_label}, ensure_ascii=False),
+                ),
+            )
+
+    return segments
+
+
+def normalize_elevenlabs_response(response: dict[str, Any]) -> list[NormalizedSegment]:
+    normalized: list[NormalizedSegment] = []
+    sequence = 0
+    for transcript in _iter_transcripts(response):
+        for speaker_label, words in _speaker_word_groups(transcript.get("words", [])):
+            segment = _build_segment(sequence, speaker_label, words, transcript.get("language_code"))
+            if segment is None:
+                continue
+            normalized.append(segment)
+            sequence += 1
+    return normalized
+
+
+def _speaker_word_groups(words: list[dict[str, Any]]) -> list[tuple[str, list[dict[str, Any]]]]:
+    groups: list[tuple[str, list[dict[str, Any]]]] = []
+    current_speaker: str | None = None
+    current_words: list[dict[str, Any]] = []
+
+    for word in words:
+        if word.get("type") == "audio_event":
+            continue
+        speaker = word.get("speaker_id") or "Unknown"
+        if current_speaker is not None and speaker != current_speaker:
+            groups.append((current_speaker, current_words))
+            current_words = []
+        current_speaker = speaker
+        current_words.append(word)
+
+    if current_speaker is not None:
+        groups.append((current_speaker, current_words))
+    return groups
+
+
+def _build_segment(
+    sequence: int,
+    speaker_label: str | None,
+    words: list[dict[str, Any]],
+    language: str | None,
+) -> NormalizedSegment | None:
+    if not words:
+        return None
+    text = "".join(str(word.get("text", "")) for word in words).strip()
+    if not text:
+        return None
+    return NormalizedSegment(
+        id=str(uuid4()),
+        sequence=sequence,
+        speaker_label=speaker_label or "Unknown",
+        text=text,
+        start_time=_first_number(words, "start"),
+        end_time=_last_number(words, "end"),
+        language=language,
+    )
+
+
+def _iter_transcripts(response: dict[str, Any]) -> list[dict[str, Any]]:
+    transcripts = response.get("transcripts")
+    if isinstance(transcripts, list):
+        return [transcript for transcript in transcripts if isinstance(transcript, dict)]
+    return [response]
+
+
+def _ensure_local_speaker(connection: Connection, meeting_id: str, provider_run_id: str, label: str) -> str:
+    row = connection.execute(
+        "SELECT id FROM local_speakers WHERE provider_run_id = ? AND label = ?",
+        (provider_run_id, label),
+    ).fetchone()
+    if row is not None:
+        return row["id"]
+
+    local_speaker_id = str(uuid4())
+    connection.execute(
+        """
+        INSERT INTO local_speakers(id, meeting_id, provider_run_id, label)
+        VALUES (?, ?, ?, ?)
+        """,
+        (local_speaker_id, meeting_id, provider_run_id, label),
+    )
+    return local_speaker_id
+
+
+def _first_number(words: list[dict[str, Any]], key: str) -> float | None:
+    for word in words:
+        value = word.get(key)
+        if isinstance(value, int | float):
+            return float(value)
+    return None
+
+
+def _last_number(words: list[dict[str, Any]], key: str) -> float | None:
+    for word in reversed(words):
+        value = word.get(key)
+        if isinstance(value, int | float):
+            return float(value)
+    return None

fly_on_the_wall/people.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from sqlite3 import Connection
+from uuid import uuid4
+
+
+@dataclass(frozen=True)
+class Person:
+    id: str
+    display_name: str
+    is_user: bool = False
+
+
+def create_person(connection: Connection, display_name: str) -> Person:
+    person = Person(id=str(uuid4()), display_name=display_name.strip(), is_user=False)
+    if not person.display_name:
+        raise ValueError("Person display name cannot be empty.")
+
+    with connection:
+        connection.execute(
+            "INSERT INTO people(id, display_name) VALUES (?, ?)",
+            (person.id, person.display_name),
+        )
+    return person
+
+
+def list_people(connection: Connection) -> list[Person]:
+    rows = connection.execute("SELECT id, display_name, is_user FROM people ORDER BY display_name").fetchall()
+    return [_person_from_row(row) for row in rows]
+
+
+def get_person(connection: Connection, person_id_or_name: str) -> Person | None:
+    row = connection.execute(
+        """
+        SELECT id, display_name, is_user FROM people
+        WHERE id = ? OR lower(display_name) = lower(?)
+        """,
+        (person_id_or_name, person_id_or_name),
+    ).fetchone()
+    if row is None:
+        return None
+    return _person_from_row(row)
+
+
+def get_user_person(connection: Connection) -> Person | None:
+    row = connection.execute("SELECT id, display_name, is_user FROM people WHERE is_user = 1 LIMIT 1").fetchone()
+    return None if row is None else _person_from_row(row)
+
+
+def set_user_person(connection: Connection, person_id_or_name: str) -> Person:
+    person = get_person(connection, person_id_or_name)
+    if person is None:
+        raise ValueError(f"Person not found: {person_id_or_name}")
+
+    with connection:
+        connection.execute("UPDATE people SET is_user = 0")
+        connection.execute(
+            "UPDATE people SET is_user = 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
+            (person.id,),
+        )
+    return Person(id=person.id, display_name=person.display_name, is_user=True)
+
+
+def unset_user_person(connection: Connection) -> Person | None:
+    person = get_user_person(connection)
+    if person is None:
+        return None
+    with connection:
+        connection.execute(
+            "UPDATE people SET is_user = 0, updated_at = CURRENT_TIMESTAMP WHERE id = ?",
+            (person.id,),
+        )
+    return Person(id=person.id, display_name=person.display_name, is_user=False)
+
+
+def _person_from_row(row) -> Person:
+    return Person(
+        id=row["id"],
+        display_name=row["display_name"],
+        is_user=bool(row["is_user"]),
+    )

fly_on_the_wall/people_embeddings.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from sqlite3 import Connection
+
+from fly_on_the_wall.embeddings import (
+    EmbeddingBackend,
+    PyannoteEmbeddingBackend,
+    cache_voice_sample_embedding,
+)
+from fly_on_the_wall.storage import StoragePaths
+
+
+@dataclass(frozen=True)
+class PeopleEmbeddingStatus:
+    people: int
+    voice_samples: int
+    embedded_voice_samples: int
+    missing_voice_sample_embeddings: int
+
+
+@dataclass(frozen=True)
+class PeopleEmbeddingBackfillResult:
+    embedded: int
+    failed: int
+
+
+def people_embedding_status(connection: Connection) -> PeopleEmbeddingStatus:
+    people = connection.execute("SELECT COUNT(*) FROM people").fetchone()[0]
+    voice = connection.execute(
+        """
+        SELECT COUNT(*) AS total,
+               SUM(CASE WHEN embedding_path IS NOT NULL THEN 1 ELSE 0 END) AS embedded
+        FROM voice_samples
+        """
+    ).fetchone()
+    total = int(voice["total"] or 0)
+    embedded = int(voice["embedded"] or 0)
+    return PeopleEmbeddingStatus(
+        people=int(people),
+        voice_samples=total,
+        embedded_voice_samples=embedded,
+        missing_voice_sample_embeddings=total - embedded,
+    )
+
+
+def backfill_people_embeddings(
+    connection: Connection,
+    storage: StoragePaths | None = None,
+    backend: EmbeddingBackend | None = None,
+) -> PeopleEmbeddingBackfillResult:
+    resolved_backend = backend or PyannoteEmbeddingBackend()
+    rows = connection.execute(
+        """
+        SELECT id FROM voice_samples
+        WHERE embedding_path IS NULL
+        ORDER BY created_at
+        """
+    ).fetchall()
+    embedded = failed = 0
+    for row in rows:
+        try:
+            cache_voice_sample_embedding(connection, row["id"], resolved_backend, storage)
+        except (FileNotFoundError, RuntimeError, ValueError):
+            failed += 1
+            continue
+        embedded += 1
+    return PeopleEmbeddingBackfillResult(embedded=embedded, failed=failed)