content-foundry 1.0.0__py3-none-any.whl

content_foundry/__init__.py

@@ -0,0 +1,4 @@
+"""Content Foundry — a resumable multi-agent pipeline that researches, writes, judges, narrates, and
+renders short-form YouTube videos for any niche."""
+
+__version__ = "1.0.0"

content_foundry/agents/__init__.py

@@ -0,0 +1,36 @@
+"""The seven agents (each owns exactly one transformation; no agent imports another)."""
+
+from __future__ import annotations
+
+from .brainstorm import Brainstormer
+from .broll_director import BrollDirector
+from .data_fetcher import DataFetcher
+from .idea_miner import IdeaMiner
+from .instruction_planner import InstructionPlanner
+from .judge import Judge
+from .publisher import Publisher
+from .renderer import Renderer
+from .research import Researcher
+from .scene_image_director import SceneImageDirector
+from .script_generator import ScriptGenerator
+from .thumbnail_director import ThumbnailDirector
+from .visuals import Visuals, build_image_prompt
+from .voiceover import Voiceover
+
+__all__ = [
+    "DataFetcher",
+    "Brainstormer",
+    "IdeaMiner",
+    "InstructionPlanner",
+    "Researcher",
+    "BrollDirector",
+    "ScriptGenerator",
+    "ThumbnailDirector",
+    "SceneImageDirector",
+    "Judge",
+    "Voiceover",
+    "Visuals",
+    "build_image_prompt",
+    "Renderer",
+    "Publisher",
+]

content_foundry/agents/brainstorm.py

@@ -0,0 +1,136 @@
+"""Agent 0 — Brainstormer. Proposes several fresh, specific, helpful video ideas from the brief so
+runs don't collapse onto the same topic. LLM-driven with a deterministic content-angle fallback."""
+
+from __future__ import annotations
+
+import json
+
+from ..errors import LLMError
+from ..logging import get_logger
+from ..models import DataBrief
+from ..prompts import load_prompt, render_prompt
+from ..providers.base import LLMProvider, extract_json
+from ..providers.tiering import TaskTier, select_model
+
+
+class Brainstormer:
+    def __init__(self, settings, llm_provider: LLMProvider):
+        self._settings = settings
+        self._llm = llm_provider
+        self._log = get_logger(component="brainstorm")
+
+    def propose(
+        self, brief: DataBrief, *, recent_ideas: list[str] | None = None, count: int = 5,
+        focus: str = "",
+    ) -> list[str]:
+        """Propose ``count`` distinct, specific video ideas. ``focus`` (e.g. the user's --idea) steers
+        every idea. The fetched data is used only as INSPIRATION / supporting stats, never the whole
+        basis. Falls back to deterministic content angles when the LLM is unavailable."""
+        recent = [r for r in (recent_ideas or []) if r]
+        fallback = self._fallback(brief, recent, focus, count)
+        try:
+            system = render_prompt(
+                load_prompt("brainstorm.system"),
+                niche=brief.niche,
+                count=str(count),
+                focus_line=(
+                    f'The viewer specifically asked for: "{focus}". EVERY idea MUST be a concrete, '
+                    "specific angle that delivers on this." if focus else ""
+                ),
+                facts_json=json.dumps([kf.statement for kf in brief.key_facts], ensure_ascii=False),
+                avoid_json=json.dumps(recent[:8], ensure_ascii=False),
+            )
+            resp = self._llm.complete(
+                "Return ONLY the JSON array now.",
+                system=system,
+                temperature=max(self._settings.llm_temperature, 0.8),
+                # A generous cap: reasoning models (e.g. Gemini flash) spend part of the budget
+                # THINKING, and a tight cap (700) left no room for the JSON -> empty/unusable reply.
+                max_tokens=self._settings.llm_max_tokens,
+                model=select_model(
+                    self._settings, TaskTier.HEAVY, fallback=self._settings.generator_model
+                ),
+            )
+            ideas = _parse_ideas(resp.text)
+            if ideas:
+                self._log.info("brainstormed_ideas", count=len(ideas))
+                return ideas[:count]
+        except (json.JSONDecodeError, LLMError, KeyError, ValueError, AttributeError, TypeError) as exc:
+            self._log.warning("brainstorm_fallback", error=str(exc))
+        return fallback
+
+    def run(self, brief: DataBrief, *, recent_ideas: list[str] | None = None) -> str:
+        """Back-compat single-idea helper."""
+        ideas = self.propose(brief, recent_ideas=recent_ideas, count=1)
+        return ideas[0] if ideas else ""
+
+    def _fallback(self, brief: DataBrief, recent: list[str], focus: str, count: int) -> list[str]:
+        """Deterministic: clean focus-based angles (or the brief's content angles when there is no
+        focus), skipping recently-made ones."""
+        pool: list[str] = []
+        if focus:
+            f = focus.strip().rstrip(".")
+            low = (f[:1].lower() + f[1:]) if f else f
+            pool += [
+                f"{f}: a practical step-by-step guide",
+                f"{f}: the mistakes that keep you stuck",
+                f"{f}: what actually works, backed by the data",
+                f"The truth about {low}",
+                f"{f}: what the {brief.niche} numbers really say",
+                f"{f} in 30 days: a realistic plan",
+                f"{f}: the first moves to make right now",
+            ]
+        pool += [a.hook for a in brief.content_angles]
+        if not pool:
+            pool = [f"A specific, data-backed {brief.niche} explainer that solves one real problem"]
+        recent_l = " ".join(recent).lower()
+        fresh = [h for h in _dedup(pool) if h[:25].lower() not in recent_l]
+        return (fresh or _dedup(pool))[:count]
+
+
+def _dedup(items) -> list[str]:
+    """Order-preserving de-dup + trim; coerces non-strings to str."""
+    seen: set[str] = set()
+    out: list[str] = []
+    for item in items:
+        text = (item if isinstance(item, str) else str(item)).strip()
+        if text and text.lower() not in seen:
+            seen.add(text.lower())
+            out.append(text)
+    return out
+
+
+def _slice_array(text: str) -> str:
+    """Slice the first ``[`` to the last ``]`` (recovers a JSON array wrapped in prose)."""
+    start, end = text.find("["), text.rfind("]")
+    return text[start : end + 1] if start != -1 and end > start else ""
+
+
+def _parse_ideas(text: str) -> list[str]:
+    """Robustly pull the list of idea strings from an LLM reply. Handles a bare JSON array, an object
+    ``{"ideas": [...]}``, an array of objects, or an array wrapped in prose. NOTE: a plain
+    ``extract_json`` MANGLES a top-level array (it slices first ``{`` to last ``}``), so we try a
+    direct parse and an array-slice too."""
+    raw = (text or "").strip()
+    data = None
+    for candidate in (raw, extract_json(raw), _slice_array(raw)):
+        if not candidate:
+            continue
+        try:
+            data = json.loads(candidate)
+            break
+        except (json.JSONDecodeError, ValueError):
+            continue
+    if isinstance(data, dict):
+        data = data.get("ideas") or data.get("items") or data.get("titles") or []
+    if not isinstance(data, list):
+        return []
+    out: list[str] = []
+    for item in data:
+        if isinstance(item, str):
+            out.append(item)
+        elif isinstance(item, dict):  # array of objects -> use the most title-like field
+            val = item.get("title") or item.get("idea") or item.get("angle") or item.get("text")
+            if isinstance(val, str):
+                out.append(val)
+    return _dedup(out)

content_foundry/agents/broll_director.py

@@ -0,0 +1,93 @@
+"""Agent 5.5 — B-roll Director. Leverages the LLM's whole-script context (e.g. Gemini) to rewrite each
+scene's B-roll search queries so the stock footage is both RELEVANT to what the narrator says AND
+visually DIVERSE across the whole video (no repeated shots). Runs in the visuals stage, gated by
+BROLL_DIRECTOR_ENABLED. Best-effort: any failure keeps the script generator's original keywords.
+"""
+
+from __future__ import annotations
+
+import json
+
+from ..errors import LLMError
+from ..logging import get_logger
+from ..models import Script
+from ..prompts import load_prompt, render_prompt
+from ..providers.base import LLMProvider, extract_json
+from ..providers.tiering import TaskTier, select_model
+
+
+class BrollDirector:
+    def __init__(self, settings, llm_provider: LLMProvider):
+        self._settings = settings
+        self._llm = llm_provider
+        self._log = get_logger(component="broll_director")
+
+    def run(self, script: Script) -> Script:
+        """Rewrite each scene's ``b_roll_keywords`` in place with relevant, diverse queries. A no-op
+        when disabled, the script has no scenes, or the LLM output is unusable."""
+        if not self._settings.broll_director_enabled or not script.scenes:
+            return script
+        try:
+            queries = self._direct(script)
+        except (json.JSONDecodeError, LLMError, ValueError, AttributeError, TypeError) as exc:
+            self._log.warning("broll_director_failed", error=str(exc))
+            return script
+        applied = 0
+        for scene in script.scenes:
+            directed = queries.get(scene.index)
+            if directed:
+                scene.b_roll_keywords = directed
+                applied += 1
+        if applied:
+            self._log.info("broll_directed", scenes=applied)
+        return script
+
+    def _direct(self, script: Script) -> dict[int, list[str]]:
+        # Hand the model the WHOLE script (every scene's narration, the on-screen stat/callout, and
+        # the generator's first-draft keywords) so it can keep each shot relevant to what is said AND
+        # deliberately distinct across scenes.
+        scenes_json = json.dumps(
+            [
+                {
+                    "index": s.index,
+                    "narration": s.narration,
+                    "on_screen": s.on_screen_text or "",
+                    "current_keywords": s.b_roll_keywords,
+                }
+                for s in script.scenes
+            ],
+            ensure_ascii=False,
+        )
+        model = select_model(
+            self._settings, TaskTier.LIGHT, fallback=self._settings.generator_model
+        )
+        system = render_prompt(
+            load_prompt("broll_director.system"),
+            max_queries=str(self._settings.broll_director_max_queries),
+            scenes_json=scenes_json,
+        )
+        resp = self._llm.complete(
+            "Return ONLY the JSON now.",
+            system=system,
+            temperature=0.4,
+            max_tokens=self._settings.llm_max_tokens,
+            model=model,
+        )
+        # The model may return a bare array or a {"scenes": [...]} object; extract_json only recovers
+        # objects, so try a direct parse first, then fall back to it.
+        try:
+            data = json.loads(resp.text.strip())
+        except json.JSONDecodeError:
+            data = json.loads(extract_json(resp.text))
+        items = data.get("scenes") if isinstance(data, dict) else data
+        out: dict[int, list[str]] = {}
+        for item in items or []:
+            if not isinstance(item, dict):
+                continue
+            idx = item.get("index")
+            queries = [
+                q.strip() for q in (item.get("queries") or []) if isinstance(q, str) and q.strip()
+            ]
+            if idx is not None and queries:
+                out[int(idx)] = queries[: self._settings.broll_director_max_queries]
+        return out

content_foundry/agents/data_fetcher.py

@@ -0,0 +1,114 @@
+"""Agent 1 — Data Fetcher. Fetch + rank signals, then deterministically distill (Ch. 7)."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from ..datasources import DataSource, build_sources
+from ..errors import DataSourceError, InsufficientDataError, NoDataError
+from ..logging import get_logger
+from ..models import DataBrief, NormalizedSignal, Provenance
+from . import distill
+
+
+class DataFetcher:
+    """Orchestrates source fetching (with cache) and deterministic distillation. No LLM."""
+
+    def __init__(self, settings, repository=None, sources: Sequence[DataSource] | None = None):
+        self._settings = settings
+        self._repo = repository
+        self._sources = sources
+        self._log = get_logger(component="data_fetcher")
+
+    def run(self, run_id: str, *, niche: str, topic_seed: str | None = None) -> DataBrief:
+        sources = list(self._sources) if self._sources is not None else build_sources(
+            self._settings, niche=niche, topic_seed=topic_seed
+        )
+        if not sources:
+            raise NoDataError("No data sources are enabled/configured.")
+
+        coverage: dict[str, bool] = {}
+        gaps: list[str] = []
+        all_signals: list[NormalizedSignal] = []
+
+        for source in sources:
+            try:
+                signals = self._fetch_with_cache(source)
+            except DataSourceError as exc:
+                self._log.warning("source_failed", source=source.name, error=str(exc))
+                coverage[source.name] = False
+                gaps.append(f"{source.name}: {exc}")
+                continue
+            coverage[source.name] = bool(signals)
+            if not signals:
+                gaps.append(f"{source.name}: no signals returned")
+            all_signals.extend(signals)
+
+        if not all_signals:
+            raise NoDataError("All data sources failed or returned nothing.")
+
+        ranked = self._rank(self._dedup(all_signals), niche=niche, topic_seed=topic_seed)
+        key_facts = distill.build_key_facts(ranked, limit=self._settings.max_facts)
+        if len(key_facts) < self._settings.min_facts:
+            raise InsufficientDataError(
+                f"Only {len(key_facts)} grounded facts (< MIN_FACTS={self._settings.min_facts})."
+            )
+        angles = distill.build_angles(ranked)
+
+        return DataBrief(
+            run_id=run_id,
+            niche=niche,
+            topic_seed=topic_seed,
+            key_facts=key_facts,
+            content_angles=angles,
+            coverage=coverage,
+            gaps=gaps,
+            provenance=Provenance(
+                produced_by="data_fetcher", model=None, config_hash=self._settings.config_hash
+            ),
+        )
+
+    # ------------------------------------------------------------------ internals
+    def _fetch_with_cache(self, source: DataSource) -> list[NormalizedSignal]:
+        ttl = self._settings.signal_cache_ttl_min
+        if self._repo is not None:
+            cached = self._repo.get_cached_signals(source.name, ttl)
+            if cached is not None:
+                self._log.info("cache_hit", source=source.name, count=len(cached))
+                return [NormalizedSignal.model_validate(c) for c in cached]
+
+        signals = source.fetch()
+        if self._repo is not None and signals:
+            self._repo.put_cached_signals(
+                source.name, [s.model_dump(mode="json") for s in signals]
+            )
+        return signals
+
+    @staticmethod
+    def _dedup(signals: Sequence[NormalizedSignal]) -> list[NormalizedSignal]:
+        seen: set[tuple] = set()
+        out: list[NormalizedSignal] = []
+        for s in signals:
+            key = (s.source, s.kind, s.title, s.value)
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(s)
+        return out
+
+    @staticmethod
+    def _rank(
+        signals: Sequence[NormalizedSignal], *, niche: str, topic_seed: str | None
+    ) -> list[NormalizedSignal]:
+        terms = [
+            w
+            for w in re.split(r"\W+", f"{niche} {topic_seed or ''}".lower())
+            if len(w) > 2
+        ]
+
+        def relevance(sig: NormalizedSignal) -> int:
+            haystack = f"{sig.title} {sig.value or ''}".lower()
+            return sum(term in haystack for term in terms)
+
+        return sorted(signals, key=lambda s: (relevance(s), s.observed_at), reverse=True)

content_foundry/agents/distill.py

@@ -0,0 +1,108 @@
+"""Deterministic distillation: NormalizedSignal -> KeyFact / ContentAngle (Ch. 7.3).
+
+NO LLM. Every field is copied from real signal data, so invented numbers are impossible.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+
+from ..models import Citation, ContentAngle, KeyFact, NormalizedSignal
+
+# Per-kind statement templates (rendered purely from structured signal fields).
+_STATEMENT_TEMPLATES = {
+    "salary": "{title} pays a median of {value} {unit}.",
+    "posting_trend": "{title}: {value} {unit}.",
+    "layoff": "{title}",
+    "news": "{title}",
+    "outlook": "{title} stands at {value} {unit}.",
+}
+
+# Per-kind content-angle hooks, filled with concrete numbers.
+_ANGLE_TEMPLATES = {
+    "salary": (
+        "What {title} really pays now ({value}) — and who is actually getting it.",
+        "Most salary advice ignores the real {value} number for {title}.",
+    ),
+    "posting_trend": (
+        "The hiring signal nobody mentions: {value} {unit}.",
+        "Why '{title}' ({value} {unit}) changes your job-search math.",
+    ),
+    "layoff": (
+        "{title} — what these layoffs actually signal for your field.",
+        "The layoff headline hides the real career move to make.",
+    ),
+    "news": (
+        "{title} — the part that affects your paycheck.",
+        "Behind the headline '{title}': the non-obvious takeaway.",
+    ),
+    "outlook": (
+        "{title} is {value} {unit} — here is what that means for you.",
+        "The outlook number ({value} {unit}) most channels get wrong.",
+    ),
+}
+
+
+def _clean(value: str | None) -> str:
+    return (value or "").strip()
+
+
+def _snippet(signal: NormalizedSignal) -> str:
+    raw_snippet = signal.raw.get("snippet") if isinstance(signal.raw, dict) else None
+    if raw_snippet:
+        return str(raw_snippet)
+    parts = [signal.title]
+    if signal.value:
+        parts.append(f"{signal.value}{(' ' + signal.unit) if signal.unit else ''}")
+    return ": ".join(p for p in parts if p)
+
+
+def build_key_fact(signal: NormalizedSignal) -> KeyFact:
+    template = _STATEMENT_TEMPLATES.get(signal.kind, "{title}")
+    statement = template.format(
+        title=_clean(signal.title), value=_clean(signal.value), unit=_clean(signal.unit)
+    ).strip()
+    statement = re.sub(r"\s+", " ", statement).strip(": ").strip() + ("" if statement.endswith(".") else "")
+    return KeyFact(
+        statement=statement,
+        metric=signal.kind,
+        value=signal.value,
+        citation=Citation(
+            source=signal.source,
+            url=signal.url,
+            observed_at=signal.observed_at,
+            snippet=_snippet(signal),
+        ),
+    )
+
+
+def build_key_facts(signals: Sequence[NormalizedSignal], *, limit: int = 8) -> list[KeyFact]:
+    return [build_key_fact(s) for s in signals[:limit]]
+
+
+def build_angles(
+    signals: Sequence[NormalizedSignal], *, limit: int = 3
+) -> list[ContentAngle]:
+    angles: list[ContentAngle] = []
+    for idx, signal in enumerate(signals):
+        if len(angles) >= limit:
+            break
+        templates = _ANGLE_TEMPLATES.get(signal.kind)
+        if not templates:
+            continue
+        hook = templates[idx % len(templates)].format(
+            title=_clean(signal.title), value=_clean(signal.value), unit=_clean(signal.unit)
+        )
+        hook = re.sub(r"\s+", " ", hook).strip()
+        angles.append(
+            ContentAngle(
+                hook=hook,
+                supporting_fact_ids=[idx],
+                why_nonobvious=(
+                    "Built directly from fetched data, not generic advice; "
+                    f"cites a concrete {signal.kind} signal from {signal.source}."
+                ),
+            )
+        )
+    return angles