devrel-origin 0.2.14__py3-none-any.whl

devrel_origin/quality/slop.py

@@ -0,0 +1,167 @@
+"""Anti-slop pipeline stage 5.
+
+Three-step matching:
+1. Regex blocklist (deterministic, fast). Word-boundary, case-insensitive.
+2. LLM lint (Haiku). Catches context-sensitive slop the regex misses
+   (verbose intros, vague intensifiers in unusual phrasings).
+3. Force-rewrite (Sonnet). One targeted rewrite call with all hits listed.
+   If the rewrite still trips the blocklist on re-check, the orchestrator
+   aborts loud, see editorial.py.
+
+Haiku's lint output occasionally hallucinates phrases that don't appear in
+the source (verbatim from a real abort: 'replace this blockquote', 'replace
+with' in a draft that contained neither). Hallucinated phrases would then go
+into force_rewrite's flagged list and re-appear on the post-rewrite re-check,
+falsely tripping the abort-loud condition. _verify_lint_hits filters out any
+phrase that does not actually appear in the text (case-insensitive substring),
+so only real slop survives into the rewrite + abort loop.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SlopHit:
+    phrase: str
+    start: int
+    end: int
+
+
+def parse_blocklist(md: str) -> list[str]:
+    """Parse `slop-blocklist.md`. Returns lowercased phrases, one per
+    non-comment, non-blank line. Lines starting with `#` are comments."""
+    out: list[str] = []
+    for raw in md.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        out.append(line.lower())
+    return out
+
+
+def find_slop(text: str, blocklist: list[str]) -> list[SlopHit]:
+    """Word-boundary, case-insensitive regex match. Returns one hit per
+    occurrence, in order."""
+    hits: list[SlopHit] = []
+    text_lower = text.lower()
+    for phrase in blocklist:
+        pattern = r"\b" + re.escape(phrase) + r"\b"
+        for m in re.finditer(pattern, text_lower):
+            hits.append(SlopHit(phrase=phrase, start=m.start(), end=m.end()))
+    hits.sort(key=lambda h: h.start)
+    return hits
+
+
+_LINT_SYSTEM = (
+    "You are an editor screening AI-written content for tells the regex "
+    "blocklist would miss: verbose intros, vague intensifiers in unusual "
+    "phrasings, hedging that doesn't appear in the blocklist verbatim. "
+    "Return a flat list, one phrase per line, lowercase, no bullets, no "
+    "explanations. If nothing concerning, return an empty response."
+)
+
+
+def _normalize_lint_lines(raw: str) -> list[str]:
+    out: list[str] = []
+    for line in raw.splitlines():
+        s = line.strip()
+        if not s or s.startswith("#"):
+            continue
+        # Strip leading bullets / ordinals.
+        s = re.sub(r"^[\-\*•\d\.\)]+\s*", "", s)
+        if s:
+            out.append(s.lower())
+    return out
+
+
+def _verify_lint_hits(text: str, candidates: list[str]) -> list[str]:
+    """Drop hallucinated lint hits (phrases Haiku flagged that don't appear).
+
+    Case-insensitive substring match. We deliberately don't require word
+    boundaries because Haiku sometimes returns slightly truncated phrases
+    (e.g. 'in essence' for the substring 'in essence,'); a substring match
+    accepts those while still rejecting fully-fabricated phrases. Logs the
+    filtered set at INFO so we can monitor Haiku's hallucination rate without
+    polluting user-facing output.
+    """
+    text_lower = text.lower()
+    real: list[str] = []
+    hallucinated: list[str] = []
+    for phrase in candidates:
+        if phrase and phrase in text_lower:
+            real.append(phrase)
+        else:
+            hallucinated.append(phrase)
+    if hallucinated:
+        logger.info(
+            "slop_lint_filtered_hallucinations",
+            extra={
+                "filtered_count": len(hallucinated),
+                "kept_count": len(real),
+                "filtered": hallucinated[:10],  # cap log size
+            },
+        )
+    return real
+
+
+async def llm_lint(text: str, voice: str, llm_client) -> list[str]:
+    """Haiku-powered second-pass slop detector.
+
+    Verifies each Haiku-flagged phrase actually appears in the source so
+    hallucinated flags don't propagate into force_rewrite or trigger the
+    orchestrator's post-rewrite abort-loud check.
+    """
+    user = (
+        "Voice contract for this product:\n\n" + (voice or "(none)") + "\n\n"
+        "Content to screen:\n\n" + text + "\n\n"
+        "List the phrases that read as AI-written, one per line. Empty if clean. "
+        "Only list phrases that appear verbatim in the content above, exactly as "
+        "they appear there; do not paraphrase, abbreviate, or invent phrases."
+    )
+    raw = await llm_client.generate(
+        system_prompt=_LINT_SYSTEM,
+        user_prompt=user,
+        model="haiku",
+    )
+    return _verify_lint_hits(text, _normalize_lint_lines(raw))
+
+
+_REWRITE_SYSTEM = (
+    "You are a rewrite editor. The reader has flagged specific phrases as "
+    "AI-written. Rewrite the content so none of the flagged phrases (or "
+    "their close synonyms) appear, while preserving meaning, structure, "
+    "and the project's voice. Return only the rewritten content — no "
+    "preamble, no explanation."
+)
+
+
+async def force_rewrite(
+    text: str,
+    regex_hits: list[SlopHit],
+    llm_lint_hits: list[str],
+    voice: str,
+    llm_client,
+) -> str:
+    """Single Sonnet rewrite with the full flagged list. Caller is
+    responsible for re-running `find_slop` + `llm_lint` to verify the
+    rewrite cleared the issues."""
+    flagged = sorted({h.phrase for h in regex_hits} | set(llm_lint_hits))
+    flagged_listing = "\n".join(f"- {p}" for p in flagged)
+    user = (
+        "Voice contract:\n\n" + (voice or "(none)") + "\n\n"
+        "Flagged phrases (do not let any of these appear in the rewrite, "
+        "and avoid close synonyms):\n\n" + flagged_listing + "\n\n"
+        "Original content:\n\n" + text
+    )
+    rewritten = await llm_client.generate(
+        system_prompt=_REWRITE_SYSTEM,
+        user_prompt=user,
+        model="sonnet",
+    )
+    return rewritten.strip()

devrel_origin/quality/style.py

@@ -0,0 +1,110 @@
+"""Load style.md and parse the per-content-type targets table.
+
+Content type names are normalized to snake_case for keying (e.g.,
+"Blog post" -> "blog_post"). Targets parsing is best-effort: malformed
+rows are skipped. If the file or table is missing, callers fall back to
+DEFAULT_TARGETS.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from devrel_origin.project.paths import ProjectPaths
+
+
+@dataclass(frozen=True)
+class ContentTypeTargets:
+    flesch_min: int
+    flesch_max: int
+    sentence_len_min: int
+    sentence_len_max: int
+    jargon_density: str
+
+
+DEFAULT_TARGETS: dict[str, ContentTypeTargets] = {
+    "tutorial": ContentTypeTargets(50, 65, 12, 18, "medium"),
+    "blog_post": ContentTypeTargets(55, 70, 12, 20, "low-medium"),
+    "landing_page": ContentTypeTargets(60, 75, 10, 15, "low"),
+    "cold_email": ContentTypeTargets(65, 80, 10, 14, "low"),
+    "battle_card": ContentTypeTargets(45, 60, 12, 18, "medium-high"),
+}
+
+
+def load_style(paths: ProjectPaths) -> str:
+    """Return the full text of `.devrel/style.md`, or "" if missing."""
+    if not paths.style_file.is_file():
+        return ""
+    return paths.style_file.read_text(encoding="utf-8")
+
+
+_RANGE_RE = re.compile(r"^\s*(\d+)\s*[–-]\s*(\d+)")
+
+
+def _parse_range(s: str) -> tuple[int, int] | None:
+    m = _RANGE_RE.match(s)
+    if not m:
+        return None
+    return int(m.group(1)), int(m.group(2))
+
+
+def _normalize_name(s: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", s.strip().lower()).strip("_")
+
+
+def parse_targets(md: str) -> dict[str, ContentTypeTargets]:
+    """Parse the per-content-type table in style.md. Looks for the first
+    pipe-table whose header row contains 'Flesch' (case-insensitive) and
+    'Jargon'. Returns a snake_case-keyed dict of ContentTypeTargets.
+    """
+    lines = md.splitlines()
+    out: dict[str, ContentTypeTargets] = {}
+    in_table = False
+    header_seen = False
+    for raw in lines:
+        line = raw.strip()
+        if not line.startswith("|"):
+            if in_table:
+                break
+            continue
+        if not header_seen:
+            lower = line.lower()
+            if "jargon" in lower and ("flesch" in lower or "f-k" in lower or "sentence" in lower):
+                header_seen = True
+                in_table = True
+            continue
+        # Skip the markdown separator row (|---|---|...).
+        if set(line.replace("|", "").strip()) <= set("- "):
+            continue
+        cells = [c.strip() for c in line.strip().strip("|").split("|")]
+        if len(cells) < 4:
+            continue
+        name_cell, flesch_cell, sentence_cell, jargon_cell = cells[:4]
+        flesch = _parse_range(flesch_cell)
+        sentence = _parse_range(sentence_cell)
+        if flesch is None or sentence is None:
+            continue
+        name = _normalize_name(name_cell)
+        if not name:
+            continue
+        out[name] = ContentTypeTargets(
+            flesch_min=flesch[0],
+            flesch_max=flesch[1],
+            sentence_len_min=sentence[0],
+            sentence_len_max=sentence[1],
+            jargon_density=jargon_cell,
+        )
+    return out
+
+
+def get_targets(content_type: str, md: str) -> ContentTypeTargets:
+    """Resolve targets for a content type: prefer parsed style.md table,
+    then fall back to DEFAULT_TARGETS. Raises KeyError if neither source
+    has the type."""
+    parsed = parse_targets(md)
+    if content_type in parsed:
+        return parsed[content_type]
+    if content_type in DEFAULT_TARGETS:
+        return DEFAULT_TARGETS[content_type]
+    raise KeyError(f"Unknown content_type: {content_type!r}")

devrel_origin/quality/voice.py

@@ -0,0 +1,15 @@
+"""Load voice.md as a single string for prompt injection."""
+
+from __future__ import annotations
+
+from devrel_origin.project.paths import ProjectPaths
+
+
+def load_voice(paths: ProjectPaths) -> str:
+    """Return the full text of `.devrel/voice.md`, or "" if the file is
+    missing. The orchestrator injects this verbatim into editorial-stage
+    system prompts as the project's voice contract.
+    """
+    if not paths.voice_file.is_file():
+        return ""
+    return paths.voice_file.read_text(encoding="utf-8")

devrel_origin/tools/__init__.py

@@ -0,0 +1,9 @@
+"""
+Tools module — API clients, GitHub integration, search, notifications, and MCP server.
+"""
+
+from devrel_origin.tools.api_client import PostHogClient
+from devrel_origin.tools.github_tools import GitHubTools
+from devrel_origin.tools.search_tools import SearchTools
+
+__all__ = ["PostHogClient", "GitHubTools", "SearchTools"]

devrel_origin/tools/analytics.py

@@ -0,0 +1,304 @@
+"""Argus data collectors — one class per source.
+
+Each collector exposes a single async method ``collect(period)`` returning
+``list[PerformanceMetric]``. Collectors do not raise — failures are logged
+and an empty list is returned, so Argus can mark the source unhealthy in
+``PerformanceReport.sources_ok`` without aborting the whole report.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+from urllib.parse import urlparse
+
+from devrel_origin.core.argus import ContentType, PerformanceMetric
+
+if TYPE_CHECKING:
+    from devrel_origin.tools.api_client import PostHogClient
+
+logger = logging.getLogger(__name__)
+
+Period = tuple[datetime, datetime]
+
+_LANDING_PATHS: frozenset[str] = frozenset(
+    {"/", "/pricing", "/about", "/contact", "/features", "/docs"}
+)
+
+
+def _classify_url(url: str) -> ContentType:
+    """Heuristic: /blog/* → blog; root + configured marketing paths → landing."""
+    path = urlparse(url).path or "/"
+    if path in _LANDING_PATHS:
+        return "landing"
+    if path.startswith("/blog/"):
+        return "blog"
+    return "landing"
+
+
+def _content_id_from_url(url: str) -> str:
+    """Stable id derived from URL path."""
+    path = urlparse(url).path or "/"
+    if path.startswith("/blog/"):
+        slug = path[len("/blog/") :].rstrip("/")
+        return f"blog/{slug}" if slug else "blog/index"
+    return path
+
+
+class PostHogCollector:
+    """Pulls page-view + unique-visitor counts from PostHog grouped by URL."""
+
+    def __init__(self, client: "PostHogClient"):
+        self.client = client
+
+    async def collect(self, period: Period) -> list[PerformanceMetric]:
+        _start, end = period
+        try:
+            rows = await self.client.fetch_events_by_url(start=_start, end=end)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("PostHogCollector failed: %s", exc)
+            return []
+
+        metrics: list[PerformanceMetric] = []
+        for row in rows:
+            url = row.get("url", "")
+            if not url:
+                continue
+            metrics.append(
+                PerformanceMetric(
+                    content_id=_content_id_from_url(url),
+                    content_type=_classify_url(url),
+                    title=row.get("title") or url,
+                    url=url,
+                    published_at=end,
+                    primary_metric=float(row.get("page_views", 0) or 0),
+                    metric_name="page_views",
+                    secondary_metrics={
+                        "unique_visitors": float(row.get("unique_visitors", 0) or 0),
+                    },
+                )
+            )
+        return metrics
+
+
+class GitHubCollector:
+    """Emits one PerformanceMetric per repo with stars_delta as primary KPI.
+
+    Wrapped client is expected to expose ``repo_full_name: str`` and
+    ``async get_repo_stats() -> dict`` with at minimum
+    ``stars, forks, open_issues, stars_delta_7d, issues_closed_7d``.
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    async def collect(self, period: Period) -> list[PerformanceMetric]:
+        _start, end = period
+        try:
+            stats = await self.client.get_repo_stats()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("GitHubCollector failed: %s", exc)
+            return []
+
+        repo = getattr(self.client, "repo_full_name", "unknown/unknown")
+        return [
+            PerformanceMetric(
+                content_id=f"repo/{repo}",
+                content_type="repo",
+                title=repo,
+                url=f"https://github.com/{repo}",
+                published_at=end,
+                primary_metric=float(stats.get("stars_delta_7d", 0) or 0),
+                metric_name="stars_delta",
+                secondary_metrics={
+                    "stars_total": float(stats.get("stars", 0) or 0),
+                    "forks": float(stats.get("forks", 0) or 0),
+                    "open_issues": float(stats.get("open_issues", 0) or 0),
+                    "issues_closed": float(stats.get("issues_closed_7d", 0) or 0),
+                },
+            )
+        ]
+
+
+def _row_in_period(row: dict, start: datetime, end: datetime) -> bool:
+    """True if row's created_at/updated_at falls in [start, end].
+
+    Falls back to True (include the row) when neither timestamp is present —
+    older Instantly campaigns may not include either field. Without this
+    filter, --since 7d and --since 90d return identical metrics.
+    """
+    raw = row.get("created_at") or row.get("updated_at")
+    if not raw:
+        return True
+    try:
+        when = datetime.fromisoformat(str(raw).replace("Z", "+00:00"))
+    except ValueError:
+        return True
+    return start <= when <= end
+
+
+class InstantlyCollector:
+    """One PerformanceMetric per email campaign; reply_rate is primary KPI.
+
+    Wrapped client is expected to expose
+    ``async list_campaigns_with_analytics() -> list[dict]`` with at minimum
+    ``id, name, sent, opens, clicks, replies, open_rate, reply_rate``,
+    and optionally ``created_at`` / ``updated_at`` for period filtering.
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    async def collect(self, period: Period) -> list[PerformanceMetric]:
+        start, end = period
+        try:
+            rows = await self.client.list_campaigns_with_analytics()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("InstantlyCollector failed: %s", exc)
+            return []
+
+        metrics: list[PerformanceMetric] = []
+        for row in rows:
+            cid = row.get("id") or ""
+            if not cid:
+                continue
+            if not _row_in_period(row, start, end):
+                continue
+            metrics.append(
+                PerformanceMetric(
+                    content_id=f"email/{cid}",
+                    content_type="email",
+                    title=row.get("name", cid),
+                    url=None,
+                    published_at=end,
+                    primary_metric=float(row.get("reply_rate", 0.0) or 0.0),
+                    metric_name="reply_rate",
+                    secondary_metrics={
+                        "sent": float(row.get("sent", 0) or 0),
+                        "opens": float(row.get("opens", 0) or 0),
+                        "clicks": float(row.get("clicks", 0) or 0),
+                        "replies": float(row.get("replies", 0) or 0),
+                        "open_rate": float(row.get("open_rate", 0.0) or 0.0),
+                    },
+                )
+            )
+        return metrics
+
+
+class SocialCollector:
+    """Reads Echo's ``social_mentions`` table, filters to ``is_own_post=1``,
+    emits one metric per post with engagement_score as the primary KPI.
+
+    Returns an empty list (and logs) if the table is missing or the period
+    yields no rows. Does not raise.
+    """
+
+    # Pinned schema contract: these columns MUST exist on Echo's
+    # social_mentions table. If a future Echo migration renames any of
+    # these, _verify_schema logs a clear warning and the collector returns
+    # [] rather than silently producing partial data.
+    _REQUIRED_COLUMNS: frozenset[str] = frozenset(
+        {
+            "platform",
+            "post_id",
+            "title",
+            "url",
+            "posted_at",
+            "upvotes",
+            "comments",
+            "engagement_score",
+            "is_own_post",
+        }
+    )
+
+    def __init__(self, state_db_path: Path):
+        self.state_db_path = state_db_path
+        self._schema_verified = False
+
+    def _verify_schema(self, conn: sqlite3.Connection) -> bool:
+        """Confirm social_mentions has all required columns.
+
+        Cached per instance via ``self._schema_verified`` so we only PRAGMA
+        once. Logs a single clear warning if columns are missing/renamed.
+        """
+        if self._schema_verified:
+            return True
+        try:
+            cols = {row[1] for row in conn.execute("PRAGMA table_info(social_mentions)")}
+        except sqlite3.OperationalError:
+            return False  # table doesn't exist yet
+        missing = self._REQUIRED_COLUMNS - cols
+        if missing:
+            logger.warning(
+                "SocialCollector: Echo's social_mentions table is missing "
+                "required columns: %s. Argus will return no social metrics "
+                "until the schema is updated.",
+                sorted(missing),
+            )
+            return False
+        self._schema_verified = True
+        return True
+
+    def _read_rows(self, start_iso: str, end_iso: str):
+        """Synchronous SQLite read; called via asyncio.to_thread to avoid
+        stalling the event loop. Returns None on any failure (the async
+        wrapper translates that to an empty result + logs)."""
+        try:
+            with sqlite3.connect(self.state_db_path) as conn:
+                conn.row_factory = sqlite3.Row
+                if not self._verify_schema(conn):
+                    return None
+                return conn.execute(
+                    "SELECT platform, post_id, title, url, posted_at, "
+                    "upvotes, comments, engagement_score "
+                    "FROM social_mentions "
+                    "WHERE is_own_post = 1 AND posted_at >= ? AND posted_at <= ?",
+                    (start_iso, end_iso),
+                ).fetchall()
+        except sqlite3.OperationalError as exc:
+            logger.info("SocialCollector: %s", exc)
+            return None
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("SocialCollector failed: %s", exc)
+            return None
+
+    async def collect(self, period: Period) -> list[PerformanceMetric]:
+        start, end = period
+        if not self.state_db_path.is_file():
+            logger.info("SocialCollector: state.db not present, skipping")
+            return []
+
+        rows = await asyncio.to_thread(
+            self._read_rows,
+            start.isoformat(),
+            end.isoformat(),
+        )
+        if rows is None:
+            return []
+
+        metrics: list[PerformanceMetric] = []
+        for row in rows:
+            try:
+                posted_at = datetime.fromisoformat(row["posted_at"].replace("Z", "+00:00"))
+            except ValueError:
+                posted_at = end
+            metrics.append(
+                PerformanceMetric(
+                    content_id=f"social/{row['platform']}/{row['post_id']}",
+                    content_type="social",
+                    title=row["title"] or row["post_id"],
+                    url=row["url"],
+                    published_at=posted_at,
+                    primary_metric=float(row["engagement_score"] or 0.0),
+                    metric_name="engagement_score",
+                    secondary_metrics={
+                        "upvotes": float(row["upvotes"] or 0),
+                        "comments": float(row["comments"] or 0),
+                    },
+                )
+            )
+        return metrics