crewforge 0.1.0__py3-none-any.whl

crewforge/recipes/content/cli.py

@@ -0,0 +1,128 @@
+"""``crewforge write`` — multi-agent article pipeline (ported from contentforge)."""
+
+from __future__ import annotations
+
+import click
+from pydantic import ValidationError
+from rich.panel import Panel
+
+from ...cli_support import (
+    EXIT_USAGE,
+    console,
+    err_console,
+    fail,
+    llm_config,
+    print_usage,
+    recipe_settings,
+    run,
+)
+from .pipeline import ContentPipeline
+from .settings import PUBLISH_TARGETS, ContentSettings
+from .text import word_count
+
+
+@click.command("write")
+@click.argument("topic")
+@click.option("--words", "-w", type=int, default=None, help="Target word count [default: 2000].")
+@click.option("--language", "-l", default=None, help="Article language [default: en].")
+@click.option("--output", "-o", default=None, help="Output directory [default: ./output].")
+@click.option(
+    "--format",
+    "-f",
+    "formats",
+    multiple=True,
+    type=click.Choice(PUBLISH_TARGETS),
+    help="Publish format (repeatable) [default: markdown].",
+)
+@click.option("--seo/--no-seo", default=None, help="Run the SEO agent [default: on].")
+@click.option("--translate", "-t", multiple=True, help="Also translate to LANG (repeatable).")
+@click.option("--threshold", type=float, default=None, help="Quality threshold 0-1 [default: 0.8].")
+@click.option("--max-iter", type=int, default=None, help="Max edit/quality rounds [default: 3].")
+@click.option(
+    "--accept-below-threshold",
+    is_flag=True,
+    help="Publish even if the final quality score stays below the threshold (warns).",
+)
+@click.pass_context
+def write(
+    ctx,
+    topic,
+    words,
+    language,
+    output,
+    formats,
+    seo,
+    translate,
+    threshold,
+    max_iter,
+    accept_below_threshold,
+):
+    """Research, outline, write, SEO-check, edit, score, translate and publish an article.
+
+    The research agent has no web access: facts come from the model's own
+    knowledge, so review the article before publishing it.
+
+    \b
+    Example:
+      crewforge write "Why unit tests matter" --words 600 -f markdown -f html
+    """
+    try:
+        settings = ContentSettings.from_mapping(recipe_settings(ctx, "content"))
+        pipe = settings.pipeline.model_dump()
+        overrides = {
+            "target_word_count": words,
+            "language": language,
+            "seo_enabled": seo,
+            "quality_threshold": threshold,
+            "max_iterations": max_iter,
+        }
+        pipe.update({k: v for k, v in overrides.items() if v is not None})
+        if formats:
+            pipe["publish_targets"] = list(formats)
+        if translate:
+            pipe["enable_translation"] = True
+            pipe["target_languages"] = list(translate)
+        if accept_below_threshold:
+            pipe["accept_below_threshold"] = True
+        settings = ContentSettings(
+            pipeline=pipe,
+            agents=settings.agents,
+            output_dir=output or settings.output_dir,
+        )
+    except ValidationError as exc:
+        first = exc.errors()[0]
+        loc = ".".join(str(p) for p in first.get("loc", ()))
+        fail(f"configuration: {loc}: {first.get('msg')}", EXIT_USAGE)
+
+    cfg = settings.pipeline
+    llm = llm_config(ctx)
+    langs = ", ".join(cfg.target_languages) if cfg.enable_translation else "off"
+    err_console.print(
+        f"[bold cyan]crewforge write[/] · {llm.provider}/{llm.model} · {cfg.target_word_count} "
+        f"words · {cfg.language} · SEO {'on' if cfg.seo_enabled else 'off'} · threshold "
+        f"{cfg.quality_threshold} · translate {langs}",
+        highlight=False,
+        soft_wrap=True,
+    )
+    result = run(ContentPipeline(llm, settings).run(topic))
+
+    if result.status == "below_threshold":
+        err_console.print(
+            f"[yellow]warning: quality score {result.quality_score:.0f} is below the threshold "
+            f"{cfg.quality_threshold * 100:.0f} (accepted with --accept-below-threshold)[/]"
+        )
+    files = "\n".join(f"  {f}" for f in result.output_files)
+    console.print(
+        Panel(
+            f"Article: {word_count(result.article)} words · quality score "
+            f"{result.quality_score:.0f}/100\nFiles:\n{files}\nMetrics: {result.metrics_path}",
+            title=f"crewforge write — {topic}",
+            border_style="green" if result.ok else "yellow",
+        ),
+        soft_wrap=True,
+    )
+    if result.tracker is not None:
+        print_usage(result.tracker)
+
+
+commands = [write]

crewforge/recipes/content/export.py

@@ -0,0 +1,84 @@
+"""Write articles to disk as Markdown (with front matter), HTML or JSON."""
+
+from __future__ import annotations
+
+import html as html_lib
+import json
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def export_markdown(content: str, metadata: dict, output_path: str | Path) -> Path:
+    """Markdown with a YAML front-matter block."""
+    import yaml
+
+    output = Path(output_path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    front = yaml.safe_dump(metadata, sort_keys=False, allow_unicode=True).strip()
+    output.write_text(f"---\n{front}\n---\n\n{content}", encoding="utf-8")
+    return output
+
+
+def _markdown_to_html(content: str) -> str:
+    """Deliberately small Markdown subset: headings, bold, italics, paragraphs."""
+    lines = []
+    for raw in content.splitlines():
+        line = html_lib.escape(raw)
+        if not line.strip():
+            continue
+        m = re.match(r"^(#{1,3}) (.+)$", line)
+        if m:
+            level = len(m.group(1))
+            lines.append(f"<h{level}>{m.group(2)}</h{level}>")
+            continue
+        line = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", line)
+        line = re.sub(r"\*(.+?)\*", r"<em>\1</em>", line)
+        lines.append(f"<p>{line}</p>")
+    return "\n        ".join(lines)
+
+
+def export_html(content: str, metadata: dict, output_path: str | Path) -> Path:
+    """Standalone semantic HTML5 page."""
+    output = Path(output_path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    esc = html_lib.escape
+    title = esc(str(metadata.get("title", "Untitled")))
+    description = esc(str(metadata.get("description", "")))
+    lang = esc(str(metadata.get("language", "en")))
+    date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    page = f"""<!DOCTYPE html>
+<html lang="{lang}">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="{description}">
+    <meta property="og:title" content="{title}">
+    <meta property="og:description" content="{description}">
+    <title>{title}</title>
+    <style>
+        body {{ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+               max-width: 800px; margin: 0 auto; padding: 2rem; line-height: 1.6; }}
+    </style>
+</head>
+<body>
+    <article>
+        {_markdown_to_html(content)}
+    </article>
+    <footer>
+        <p><small>Generated {date} with crewforge (content recipe).
+        Review before publishing.</small></p>
+    </footer>
+</body>
+</html>
+"""
+    output.write_text(page, encoding="utf-8")
+    return output
+
+
+def export_json_report(data: dict[str, Any], output_path: str | Path) -> Path:
+    output = Path(output_path)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+    return output

crewforge/recipes/content/pipeline.py

@@ -0,0 +1,222 @@
+"""Content pipeline: research → outline → write → SEO → (edit ↔ quality) → translate → publish.
+
+Every step runs through ``Crew.step``, so an agent that errors or returns
+nothing stops the run naming itself — contentforge never checked the writer,
+SEO, editor or translator results. A quality score that never reaches the
+threshold is also a failure unless ``accept_below_threshold`` is set.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+from ...core.client import LLMClient
+from ...core.config import LLMConfig
+from ...core.crew import Crew
+from ...core.errors import AgentFailure, CrewFailedError
+from ...core.usage import UsageTracker
+from .agents import AGENT_REGISTRY, AgentResult, ContentAgent
+from .settings import ContentSettings
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PipelineResult:
+    """Final result from the complete pipeline."""
+
+    status: str = "success"  # success | below_threshold
+    article: str = ""
+    research: str = ""
+    outline: str = ""
+    seo_analysis: str = ""
+    quality_report: str = ""
+    quality_score: float | None = None
+    publisher_output: str = ""
+    output_files: list[str] = field(default_factory=list)
+    translations: dict[str, str] = field(default_factory=dict)
+    agent_results: dict[str, AgentResult] = field(default_factory=dict)
+    metrics_path: str = ""
+    total_tokens: int = 0
+    pipeline_duration_s: float = 0.0
+    tracker: UsageTracker | None = field(default=None, repr=False)
+
+    @property
+    def ok(self) -> bool:
+        return self.status == "success"
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "status": self.status,
+            "total_tokens": self.total_tokens,
+            "pipeline_duration_s": round(self.pipeline_duration_s, 2),
+            "agents_run": list(self.agent_results.keys()),
+            "article_length": len(self.article),
+            "word_count": len(self.article.split()),
+            "quality_score": self.quality_score,
+            "translations": list(self.translations.keys()),
+            "output_files": self.output_files,
+            "metrics_path": self.metrics_path,
+        }
+
+
+class ContentPipeline:
+    """Coordinates the eight content agents on one crew."""
+
+    AGENT_ORDER = list(AGENT_REGISTRY)
+
+    def __init__(
+        self,
+        llm: LLMConfig,
+        settings: ContentSettings | None = None,
+        *,
+        client: LLMClient | None = None,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ) -> None:
+        self.llm = llm
+        self.settings = settings or ContentSettings()
+        self._client = client
+        self._transport = transport
+
+    def _agent(self, crew: Crew, name: str) -> ContentAgent:
+        return AGENT_REGISTRY[name](crew.client, crew.tracker, config=self.settings)
+
+    async def _step(self, crew: Crew, result: PipelineResult, key: str, agent, **kwargs):
+        out: AgentResult = await crew.step(key, agent.execute(**kwargs))
+        if not out.content.strip():  # defence in depth; ask() already refuses blanks
+            failure = AgentFailure(key, "returned empty output", "empty")
+            raise CrewFailedError(crew.name, [failure])
+        result.agent_results[key] = out
+        return out
+
+    async def run(self, topic: str) -> PipelineResult:
+        cfg = self.settings.pipeline
+        result = PipelineResult()
+        crew = Crew("content", self.llm, client=self._client, transport=self._transport)
+        try:
+            async with crew:
+                research = await self._step(
+                    crew, result, "research", self._agent(crew, "research"), topic=topic
+                )
+                result.research = research.content
+
+                outline = await self._step(
+                    crew,
+                    result,
+                    "outline",
+                    self._agent(crew, "outline"),
+                    research_data=research.content,
+                    target_words=cfg.target_word_count,
+                )
+                result.outline = outline.content
+
+                draft = await self._step(
+                    crew,
+                    result,
+                    "writer",
+                    self._agent(crew, "writer"),
+                    outline=outline.content,
+                    research_data=research.content,
+                    language=cfg.language,
+                )
+                current = draft.content
+
+                seo_notes = ""
+                if cfg.seo_enabled:
+                    seo = await self._step(
+                        crew, result, "seo", self._agent(crew, "seo"), content=current
+                    )
+                    result.seo_analysis = seo_notes = seo.content
+
+                threshold = cfg.quality_threshold * 100
+                for iteration in range(cfg.max_iterations):
+                    edited = await self._step(
+                        crew,
+                        result,
+                        "editor",
+                        self._agent(crew, "editor"),
+                        content=current,
+                        seo_recommendations=seo_notes,
+                    )
+                    current = edited.content
+                    quality = await self._step(
+                        crew,
+                        result,
+                        "quality",
+                        self._agent(crew, "quality"),
+                        content=current,
+                        threshold=cfg.quality_threshold,
+                        title=topic,
+                    )
+                    result.quality_report = quality.content
+                    result.quality_score = quality.metadata["overall_score"]
+                    if result.quality_score >= threshold:
+                        break
+                    logger.info(
+                        "[content] quality %.0f < %.0f (iteration %d/%d)",
+                        result.quality_score,
+                        threshold,
+                        iteration + 1,
+                        cfg.max_iterations,
+                    )
+                result.article = current
+
+                if result.quality_score is not None and result.quality_score < threshold:
+                    message = (
+                        f"final quality score {result.quality_score:.0f} is below the "
+                        f"threshold {threshold:.0f} after {cfg.max_iterations} edit round(s)"
+                    )
+                    if not cfg.accept_below_threshold:
+                        failure = AgentFailure("quality", message, "below_threshold")
+                        crew.failures.append(failure)
+                        raise CrewFailedError(crew.name, [failure])
+                    result.status = "below_threshold"
+
+                if cfg.enable_translation and cfg.target_languages:
+                    work = {
+                        f"translator_{lang}": self._agent(crew, "translator").execute(
+                            content=current, target_language=lang, source_language=cfg.language
+                        )
+                        for lang in cfg.target_languages
+                    }
+                    for key, out in (await crew.parallel(work)).items():
+                        if not out.content.strip():
+                            raise CrewFailedError(
+                                crew.name, [AgentFailure(key, "returned empty output", "empty")]
+                            )
+                        result.agent_results[key] = out
+                        result.translations[key.removeprefix("translator_")] = out.content
+
+                publisher = await self._step(
+                    crew,
+                    result,
+                    "publisher",
+                    self._agent(crew, "publisher"),
+                    content=current,
+                    title=topic,
+                    publish_targets=cfg.publish_targets,
+                    output_dir=self.settings.output_dir,
+                    language=cfg.language,
+                )
+                result.publisher_output = publisher.content
+                result.output_files = list(publisher.metadata.get("output_files", []))
+                out_dir = Path(self.settings.output_dir)
+                for lang, text in result.translations.items():
+                    path = out_dir / f"translation.{lang}.md"
+                    path.write_text(text, encoding="utf-8")
+                    result.output_files.append(str(path))
+        finally:
+            result.tracker = crew.tracker
+            result.total_tokens = crew.tracker.total_tokens
+            result.pipeline_duration_s = crew.tracker.duration_s
+        metrics = Path(self.settings.output_dir) / "metrics" / f"{crew.tracker.run_id}.json"
+        result.metrics_path = str(crew.tracker.save(metrics))
+        return result
+
+    def get_agent_list(self) -> list[dict[str, str]]:
+        return [{"name": n, "description": c.description} for n, c in AGENT_REGISTRY.items()]

crewforge/recipes/content/settings.py

@@ -0,0 +1,63 @@
+"""Settings for the content recipe (the ``content:`` section of crewforge.yaml)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field, field_validator
+
+PUBLISH_TARGETS = ("markdown", "html", "json")
+
+
+class PipelineConfig(BaseModel):
+    topic: str = ""
+    target_word_count: int = 2000
+    language: str = "en"
+    seo_enabled: bool = True
+    quality_threshold: float = 0.8  # fraction of 100
+    max_iterations: int = 3  # editor <-> quality rounds
+    accept_below_threshold: bool = False
+    enable_translation: bool = False
+    target_languages: list[str] = Field(default_factory=lambda: ["zh", "ms"])
+    publish_targets: list[str] = Field(default_factory=lambda: ["markdown"])
+
+    @field_validator("quality_threshold")
+    @classmethod
+    def _threshold_range(cls, v: float) -> float:
+        if not 0 <= v <= 1:
+            raise ValueError("quality_threshold is a fraction between 0 and 1")
+        return v
+
+    @field_validator("max_iterations")
+    @classmethod
+    def _iterations_positive(cls, v: int) -> int:
+        if v < 1:
+            raise ValueError("max_iterations must be >= 1")
+        return v
+
+
+class AgentConfig(BaseModel):
+    """Per-agent overrides (model, sampling, system prompt)."""
+
+    name: str
+    enabled: bool = True
+    model_override: str | None = None
+    temperature_override: float | None = None
+    max_tokens_override: int | None = None
+    system_prompt_override: str | None = None
+
+
+class ContentSettings(BaseModel):
+    pipeline: PipelineConfig = Field(default_factory=PipelineConfig)
+    agents: list[AgentConfig] = Field(default_factory=list)
+    output_dir: str = "./output"
+
+    @classmethod
+    def from_mapping(cls, data: dict[str, Any] | None) -> ContentSettings:
+        return cls(**(data or {}))
+
+    def get_agent_config(self, name: str) -> AgentConfig:
+        for agent in self.agents:
+            if agent.name == name:
+                return agent
+        return AgentConfig(name=name)

crewforge/recipes/content/text.py

@@ -0,0 +1,57 @@
+"""Text helpers for the content recipe.
+
+JSON extraction lives in the core (:func:`crewforge.core.extract_json`); it is
+re-exported here because contentforge exposed it from its text utilities.
+"""
+
+from __future__ import annotations
+
+import re
+
+from ...core.parsing import extract_json
+
+__all__ = [
+    "extract_frontmatter",
+    "extract_json",
+    "reading_time",
+    "slugify",
+    "truncate",
+    "word_count",
+]
+
+
+def word_count(text: str) -> int:
+    """Count words; each CJK character counts as one word."""
+    cjk = len(re.findall(r"[一-鿿㐀-䶿]", text))
+    english_words = len(re.findall(r"[a-zA-Z]+", text))
+    return cjk + english_words
+
+
+def extract_frontmatter(text: str) -> tuple[dict, str]:
+    """Split YAML front matter from a Markdown document."""
+    if text.startswith("---"):
+        parts = text.split("---", 2)
+        if len(parts) >= 3:
+            import yaml
+
+            fm = yaml.safe_load(parts[1]) or {}
+            return fm, parts[2].strip()
+    return {}, text
+
+
+def truncate(text: str, max_chars: int = 100, suffix: str = "...") -> str:
+    if len(text) <= max_chars:
+        return text
+    return text[: max_chars - len(suffix)] + suffix
+
+
+def slugify(text: str) -> str:
+    text = text.lower().strip()
+    text = re.sub(r"[^\w\s-]", "", text)
+    text = re.sub(r"[-\s]+", "-", text)
+    return text.strip("-")
+
+
+def reading_time(text: str, wpm: int = 200) -> str:
+    minutes = max(1, round(word_count(text) / wpm))
+    return f"{minutes} min read"

crewforge/recipes/docs/__init__.py

@@ -0,0 +1,48 @@
+"""``docs`` recipe (from documind): ingest documents, then answer questions about
+them with ``[Source N]`` citations, optional fact-checking, and summaries.
+
+Ingestion, indexing and keyword retrieval are deterministic (no LLM calls);
+the answerer, fact-checker, summarizer, reranker and reference formatter call
+the LLM through :class:`crewforge.core.Agent`.
+"""
+
+from .agents import (
+    Answer,
+    AnswerAgent,
+    CitationAgent,
+    FactCheckAgent,
+    FactCheckResult,
+    FormattedCitation,
+    RerankAgent,
+    SummarizerAgent,
+    Summary,
+)
+from .export import ExportAgent
+from .index import IndexAgent, IndexEntry
+from .ingest import Document, IngestAgent, TextChunk
+from .pipeline import DocsPipeline, PipelineResult
+from .retrieve import RetrievalResult, RetrieverAgent
+from .settings import DocsSettings
+
+__all__ = [
+    "Answer",
+    "AnswerAgent",
+    "CitationAgent",
+    "DocsPipeline",
+    "DocsSettings",
+    "Document",
+    "ExportAgent",
+    "FactCheckAgent",
+    "FactCheckResult",
+    "FormattedCitation",
+    "IndexAgent",
+    "IndexEntry",
+    "IngestAgent",
+    "PipelineResult",
+    "RerankAgent",
+    "RetrievalResult",
+    "RetrieverAgent",
+    "SummarizerAgent",
+    "Summary",
+    "TextChunk",
+]