mediasense 0.4.1__py3-none-any.whl

mediasense/__init__.py

@@ -0,0 +1,5 @@
+"""mediasense — transcribe audio/video with word-level timestamps and edit it by text."""
+
+__version__ = "0.4.1"
+
+from .models import Segment, Transcript, Word  # noqa: E402,F401

mediasense/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

mediasense/_skill/SKILL.md

@@ -0,0 +1,84 @@
+---
+name: mediasense
+description: Watch and listen to videos and audio (TikTok, YouTube, Instagram, X, Vimeo links or files) to summarize, explain, transcribe or edit them with the mediasense tools instead of opening the page.
+---
+
+# mediasense
+
+mediasense runs on the user's computer and gives you ears and eyes for media: it downloads links,
+transcribes speech (any language, timestamp per word), picks key frames, and edits audio/video by text.
+
+## First: use the tools, not the web
+
+When the user pastes a TikTok / YouTube / Instagram / X / Vimeo link or points at a media file:
+
+- **Do not** WebFetch the page or open it in a browser. Those sites block bots, and the page has no
+  transcript anyway. Go straight to the mediasense tools.
+- The tools are named `mediasense` → `understand_media`, `transcribe`, `check_job`, … In some apps they
+  are listed but not loaded yet: search your tools for "mediasense" and load them.
+- If no mediasense tool exists at all, mediasense isn't installed or Claude Desktop wasn't restarted.
+  Say so and point to https://github.com/joaoestella/mediasense#setup (quit Claude Desktop from the tray
+  icon and reopen it after installing). In a terminal you can also run the CLI: `mediasense --help`.
+
+## Golden rules
+
+- **Long jobs.** `understand_media`, `transcribe`, `render_edit` and `watch_video` reply
+  `STILL WORKING — job <id>` when they need more than ~30 s (long videos on a CPU; the first run also
+  downloads the ~500 MB speech model). Tell the user it's processing, with the percentage, then call
+  `check_job(job_id)` until the result arrives. Don't start the request again.
+- **Never modify the original.** Every output is a new file (`<name>.edited.mp4`, `<name>.placed.mp4`…).
+- **Preview before cutting.** Run `plan_edit` (CLI: `mediasense edit … --dry-run`), show the user the cut
+  list in plain words ("I'll remove 14 'um's and shorten 6 pauses, 2:31 → 2:12"), then `render_edit`.
+- **Transcribe once.** The transcript is saved as `<name>.transcript.json` next to the media and every
+  other tool finds it automatically.
+- Times are seconds (`75.5`) or `m:ss.s` (`1:15.5`). Quote timestamps back to the user as `m:ss`.
+
+## Workflows
+
+### "Summarize this" / "What is this video about?" / a pasted link
+1. `understand_media(source)` with the link or file path. One call returns the post metadata, the
+   transcript and, for videos, contact sheets of key frames with what is said per shot.
+2. Pass `vocabulary` with names you can infer (title, hashtags, the user's message) and `language` if
+   known.
+3. **Cross-check speech with the screen.** Speech recognition mangles names and tickers ("YouTube 4"
+   for ITUB4, "Cloud Fair" for Cloudflare). Fix them using the title, description and on-screen text in
+   the contact sheets. Use `get_frame(path, "m:ss")` to read small text.
+4. Answer in the user's language, citing times (`at 0:42 he shows…`).
+
+### "When does he say X?"
+`search_transcript(path, "X")` → list of start/end times. Case, accents and punctuation are ignored.
+
+### Cutting by text
+- Remove a phrase: `cut_phrases=["off the record"]` (all occurrences; `occurrence=2` for only the 2nd).
+- Remove a passage: `cut_between=[["first words of it", "last words of it"]]`.
+- Remove fillers: transcribe with `verbatim=true` first (Whisper otherwise hides most "um"s), then
+  `fillers=true`. Context-dependent words ("like", "tipo", "né") only via `filler_words` after asking.
+- Tighten pauses: `silences=0.7` (pauses over 0.7 s shrink to `silence_keep`, default 0.2 s). With music in
+  the background use `silence_source="transcript"`.
+- Raw times: `cut_ranges=["1:02-1:05.5"]`, `keep_only="0:30-2:00"`.
+- For people who edit in Premiere/Resolve/Final Cut, add `edl=true` or `fcpxml=true` so they can keep
+  editing on their own timeline. `srt=true` gives subtitles matching the edited version.
+
+### Subtitles
+`export_transcript(path, ["srt"])`; `words_per_cue=2` for TikTok/Reels-style captions.
+
+### Putting audio into a video
+1. Transcribe the video and use `search_transcript` / `understand_media` to find the moment.
+2. `place_audio(base=video, insert=audio, at="1:23.4", mode="duck")`: `mix`, `duck` (lowers the original)
+   or `replace`. `insert_from`/`insert_to` use only a slice of the clip.
+
+### "What happens in this video?"
+1. `understand_media(source)`.
+2. Look at the contact sheets: each tile shows shot number and start time; the text lists what is said per
+   shot. Describe the video chronologically with timestamps; `get_frame(path, "2:10")` to inspect detail.
+
+## Choosing a model
+`tiny`/`base`: fast drafts · `small` (default): good balance · `medium`/`large-v3`: best accuracy, slow on
+CPU · `turbo`: near large-v3 quality, faster. A CUDA GPU is used automatically when its libraries exist.
+`backend="whisperx"` gives tighter word boundaries (best for precise cuts) and speaker labels.
+
+## Troubleshooting
+- A link fails to download: sites change often; the user can update with
+  `python -m pipx runpip mediasense install -U yt-dlp`, or set `MEDIASENSE_COOKIES_FROM_BROWSER=firefox`.
+- "ffmpeg not found": the user must install FFmpeg (`winget install ffmpeg` / `brew install ffmpeg`).
+- Anything else: ask the user to run `mediasense doctor` and paste the output.

mediasense/api.py

@@ -0,0 +1,278 @@
+"""High-level operations shared by the CLI and the MCP server."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+
+from . import export
+from .edit import EditPlan, PlanBuilder, remap_transcript
+from .media import MediaInfo, detect_silences, probe, render_keep
+from .models import Transcript, parse_time
+
+Range = tuple[float, float]
+
+
+def default_transcript_path(media: str | Path) -> Path:
+    p = Path(media)
+    return p.with_name(p.stem + ".transcript.json")
+
+
+def load_transcript_for(media: str | Path | None, transcript: str | Path | None = None,
+                        required: bool = False) -> Transcript | None:
+    candidates = [Path(transcript)] if transcript else []
+    if media is not None and not transcript:
+        candidates.append(default_transcript_path(media))
+    for c in candidates:
+        if c.exists():
+            return Transcript.load(c)
+    if transcript:
+        raise FileNotFoundError(f"transcript not found: {transcript}")
+    if required:
+        raise FileNotFoundError(
+            f"no transcript for {media}; run `mediasense transcribe {media}` first"
+        )
+    return None
+
+
+def parse_range(text: str) -> Range:
+    """"1:02-1:05.5", "62..65.5" or "62,65.5" -> (62.0, 65.5)."""
+    for sep in ("..", ",", "-"):
+        if sep in text:
+            a, b = text.split(sep, 1)
+            return parse_time(a), parse_time(b)
+    raise ValueError(f"cannot parse range '{text}' (use START-END, e.g. 1:02-1:05)")
+
+
+def plan_edit(
+    media: str | Path,
+    transcript: Transcript | None = None,
+    cut_phrases: Iterable[str] = (),
+    cut_between: Iterable[tuple[str, str]] = (),
+    cut_ranges: Iterable[Range] = (),
+    keep_only: Range | None = None,
+    fillers: bool = False,
+    filler_words: Iterable[str] = (),
+    silences: float | None = None,
+    silence_keep: float = 0.2,
+    silence_source: str = "audio",
+    noise_db: float = -35.0,
+    occurrence: int | None = None,
+    pad: float = 0.12,
+) -> tuple[EditPlan, MediaInfo, dict]:
+    """Build an edit plan. Returns (plan, media info, report of what matched)."""
+    info = probe(media)
+    b = PlanBuilder(info.duration, transcript, max_pad=pad)
+    report: dict = {"not_found": []}
+    for phrase in cut_phrases:
+        hits = b.remove_phrase(phrase, occurrence)
+        if not hits:
+            report["not_found"].append(phrase)
+    for start_p, end_p in cut_between:
+        if b.remove_between(start_p, end_p) is None:
+            report["not_found"].append(f"{start_p} … {end_p}")
+    for a, e in cut_ranges:
+        b.remove_range(a, e)
+    if keep_only:
+        b.keep_only(*keep_only)
+    if fillers or list(filler_words):
+        report["fillers"] = b.remove_fillers(filler_words)
+    if silences is not None:
+        if silence_source == "transcript":
+            gaps = b.transcript_gaps(silences)
+        else:
+            gaps = detect_silences(media, noise_db=noise_db, min_duration=silences)
+        report["silences"] = b.remove_silences(gaps, keep=silence_keep)
+    return b.build(), info, report
+
+
+def apply_plan(
+    media: str | Path,
+    plan: EditPlan,
+    info: MediaInfo,
+    transcript: Transcript | None = None,
+    output: str | Path | None = None,
+    edl: str | Path | None = None,
+    fcpxml: str | Path | None = None,
+    subtitles: str | Path | None = None,
+    transcript_out: str | Path | None = None,
+) -> dict[str, str]:
+    """Render and/or export. Returns {kind: path} of everything written."""
+    written: dict[str, str] = {}
+    keep = plan.keep
+    if output:
+        render_keep(media, keep, output)
+        written["media"] = str(output)
+    if edl:
+        export.write(edl, export.to_edl(keep, info, title=Path(media).stem))
+        written["edl"] = str(edl)
+    if fcpxml:
+        export.write(fcpxml, export.to_fcpxml(keep, info, project=Path(media).stem + " (mediasense)"))
+        written["fcpxml"] = str(fcpxml)
+    if transcript and (subtitles or transcript_out):
+        edited = remap_transcript(transcript, plan)
+        if subtitles:
+            write_subtitles(edited, subtitles)
+            written["subtitles"] = str(subtitles)
+        if transcript_out:
+            edited.save(transcript_out)
+            written["transcript"] = str(transcript_out)
+    return written
+
+
+def write_subtitles(transcript: Transcript, path: str | Path, words_per_cue: int | None = None,
+                    max_chars: int = 42) -> Path:
+    path = Path(path)
+    ext = path.suffix.lower()
+    if ext == ".srt":
+        content = export.to_srt(export.subtitle_cues(transcript, max_chars=max_chars, words_per_cue=words_per_cue))
+    elif ext == ".vtt":
+        content = export.to_vtt(export.subtitle_cues(transcript, max_chars=max_chars, words_per_cue=words_per_cue))
+    elif ext == ".txt":
+        content = export.to_text(transcript)
+    elif ext == ".csv":
+        content = export.to_words_csv(transcript)
+    elif ext == ".json":
+        return transcript.save(path)
+    else:
+        raise ValueError(f"unsupported export format '{ext}' (use .srt, .vtt, .txt, .csv or .json)")
+    return export.write(path, content)
+
+
+def outputs_for(media: str | Path, suffix: str = "edited") -> dict[str, Path]:
+    p = Path(media)
+    base = f"{p.stem}.{suffix}"
+
+    def named(ext: str) -> Path:
+        # build names explicitly: Path.with_suffix would swap ".edited" for the extension
+        return p.with_name(base + ext)
+
+    return {
+        "media": named(p.suffix),
+        "edl": named(".edl"),
+        "fcpxml": named(".fcpxml"),
+        "srt": named(".srt"),
+        "transcript": named(".transcript.json"),
+    }
+
+
+def transcript_excerpt(t: Transcript, start: float | None = None, end: float | None = None,
+                       words: bool = False, max_chars: int = 20000) -> str:
+    """Compact, model-friendly view with timestamps."""
+    from .models import fmt_time
+
+    lines: list[str] = []
+    for s in t.segments:
+        if start is not None and s.end < start:
+            continue
+        if end is not None and s.start > end:
+            break
+        who = f"{s.speaker}: " if s.speaker else ""
+        if words:
+            ws = " ".join(f"{w.text.strip()}@{w.start:.2f}" for w in s.words
+                          if (start is None or w.end >= start) and (end is None or w.start <= end))
+            lines.append(f"[{fmt_time(s.start)}–{fmt_time(s.end)}] {who}{ws}")
+        else:
+            lines.append(f"[{fmt_time(s.start)}–{fmt_time(s.end)}] {who}{s.text.strip()}")
+    text = "\n".join(lines)
+    if len(text) > max_chars:
+        text = text[:max_chars] + f"\n… (truncated; ask for a time range, total {fmt_time(t.duration or 0)})"
+    return text
+
+
+def split_ranges(values: Sequence[str]) -> list[Range]:
+    return [parse_range(v) for v in values]
+
+
+def understand(
+    source: str,
+    language: str | None = None,
+    model: str = "small",
+    vocabulary: list[str] | str | None = None,
+    max_frames: int = 24,
+    backend: str = "faster-whisper",
+    force: bool = False,
+    progress=None,
+) -> dict:
+    """Everything a model needs to understand a file or link, in one call:
+    metadata, transcript with timestamps and (for video) a storyboard of shots.
+
+    Reuses the transcript/storyboard already on disk unless ``force``.
+    """
+    from .fetch import is_url, read_info, resolve
+    from .transcribe import transcribe
+
+    say = progress or (lambda _m: None)
+    if is_url(source):
+        say("opening the link (downloads the video the first time)…")
+    src = resolve(source)
+    info = probe(src)
+    meta = read_info(src)
+    result: dict = {"path": str(src), "info": info.to_dict(), "metadata": meta}
+
+    transcript = None
+    tpath = default_transcript_path(src)
+    if info.has_audio:
+        if tpath.exists() and not force:
+            transcript = Transcript.load(tpath)
+        else:
+            transcript = transcribe(src, backend=backend, model=model, language=language,
+                                    vocabulary=vocabulary, progress=say)
+            transcript.save(tpath)
+        result["transcript_path"] = str(tpath)
+    result["transcript"] = transcript
+
+    if info.has_video:
+        from .video import storyboard
+
+        out = Path(src).with_name(Path(src).stem + ".storyboard")
+        sb_json = out / "storyboard.json"
+        cached = None
+        if sb_json.exists() and not force:
+            import json
+
+            cached = json.loads(sb_json.read_text(encoding="utf-8"))
+            # a storyboard made before the transcript existed has no speech: rebuild it
+            if transcript is not None and not any(s.get("speech") for s in cached.get("shots", [])):
+                cached = None
+        if cached:
+            result["storyboard"] = cached
+        else:
+            say("picking key frames…")
+            result["storyboard"] = storyboard(src, out, transcript, max_frames=max_frames)
+        result["storyboard_dir"] = str(out)
+    return result
+
+
+def understand_report(result: dict, max_chars: int = 15000) -> str:
+    """Model-friendly text version of :func:`understand`."""
+    from .models import fmt_time
+
+    info = result["info"]
+    lines = [f"file: {result['path']}"]
+    kind = "video" if info["has_video"] else "audio"
+    extra = f" · {info['width']}x{info['height']}" if info["has_video"] else ""
+    lines.append(f"{kind} · {fmt_time(info['duration'])}{extra}")
+    meta = result.get("metadata") or {}
+    for key in ("title", "uploader", "webpage_url", "upload_date"):
+        if meta.get(key):
+            lines.append(f"{key}: {meta[key]}")
+    if meta.get("description"):
+        lines.append(f"description: {str(meta['description'])[:600]}")
+    t: Transcript | None = result.get("transcript")
+    if t is not None:
+        lines += ["", f"## Transcript (language: {t.language}, {len(t.words())} words)",
+                  transcript_excerpt(t, max_chars=max_chars)]
+    elif not info["has_audio"]:
+        lines += ["", "(no audio track)"]
+    sb = result.get("storyboard")
+    if sb:
+        sheets = [str(Path(result["storyboard_dir"]) / name) for name in sb.get("contact_sheets", [])]
+        if sheets:
+            lines += ["", "## Key frames",
+                      "Contact sheets (open these images to see the video; each tile is labelled #shot and time):"]
+            lines += [f"- {p}" for p in sheets]
+        lines += ["", f"## Shots ({len(sb['shots'])}; contact sheets show #number and start time)"]
+        for s in sb["shots"]:
+            lines.append(f"#{s['index']} {s['start_tc']}–{s['end_tc']}: {s['speech'] or '—'}")
+    return "\n".join(lines)