pmkit 0.1.1__py3-none-any.whl

pmkit/connectors/__init__.py

@@ -0,0 +1,35 @@
+"""Discovery connectors registry.
+
+Each connector fetches signals from one OSS source and returns normalized candidate
+dicts with provenance. Connectors degrade gracefully: a missing key or a source error
+skips that source rather than aborting the run.
+"""
+
+from __future__ import annotations
+
+from .github import GitHubConnector
+from .hn import HNConnector
+from .reddit import RedditConnector
+from .web import WebConnector
+from .x import XConnector
+
+# Order matters only for display; discovery runs all available connectors.
+REGISTRY = [
+    GitHubConnector(),
+    HNConnector(),
+    RedditConnector(),
+    WebConnector(),
+    XConnector(),
+]
+
+
+def get_connectors(names=None):
+    """Return connector instances, optionally filtered by name."""
+    if not names:
+        return list(REGISTRY)
+    wanted = set(names)
+    selected = [c for c in REGISTRY if c.name in wanted]
+    unknown = wanted - {c.name for c in REGISTRY}
+    if unknown:
+        raise ValueError(f"unknown source(s): {sorted(unknown)}")
+    return selected

pmkit/connectors/base.py

@@ -0,0 +1,67 @@
+"""Connector framework: config, HTTP helper, and the candidate shape.
+
+A connector fetches signals from one OSS source and returns *candidate* dicts. The HTTP
+fetch is always separated from a pure ``parse_*`` function so parsing is unit-testable
+without the network. Connectors never raise out of discovery: a missing key or a source
+error is reported as a skip, not a crash (graceful degradation, R3/U3).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Optional
+
+DEFAULT_UA = "pmkit/0.1 (+https://github.com/dkedar7/pm-system)"
+
+
+class ConnectorError(Exception):
+    """A source-level failure (auth, network, bad target). Caught by discovery."""
+
+
+@dataclass
+class Config:
+    """Runtime config, sourced from the environment. All keys optional."""
+
+    github_token: Optional[str] = None
+    brave_key: Optional[str] = None
+    x_bearer: Optional[str] = None
+    user_agent: str = DEFAULT_UA
+    timeout: float = 15.0
+    min_engagement: int = 2  # below this, a candidate is flagged low-confidence
+
+    @classmethod
+    def from_env(cls) -> "Config":
+        return cls(
+            github_token=os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN"),
+            brave_key=os.environ.get("BRAVE_API_KEY"),
+            x_bearer=os.environ.get("X_BEARER_TOKEN") or os.environ.get("TWITTER_BEARER_TOKEN"),
+        )
+
+
+def candidate(title: str, problem: str, source_type: str, url: str,
+              engagement: int = 0, created_at: Optional[str] = None) -> dict:
+    """Build a normalized candidate dict with one source of provenance."""
+    return {
+        "title": (title or "").strip()[:200],
+        "problem": (problem or "").strip()[:1000],
+        "engagement": int(engagement or 0),
+        "source": {"type": source_type, "url": url, "created_at": created_at},
+    }
+
+
+def http_get_json(url: str, headers: Optional[dict] = None, timeout: float = 15.0):
+    """GET a URL and parse JSON. Raises ConnectorError on any failure."""
+    req = urllib.request.Request(url, headers={"User-Agent": DEFAULT_UA, **(headers or {})})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        raise ConnectorError(f"HTTP {e.code} for {url}") from e
+    except (urllib.error.URLError, TimeoutError) as e:
+        raise ConnectorError(f"network error for {url}: {e}") from e
+    except json.JSONDecodeError as e:
+        raise ConnectorError(f"bad JSON from {url}: {e}") from e

pmkit/connectors/changelog.py

@@ -0,0 +1,37 @@
+"""Changelog / releases utility.
+
+Recent releases are not opportunities, so this is not a candidate-producing connector.
+It supplies "recently shipped" context that the already-solved kill-test (U4) uses to
+refute candidates a maintainer has just addressed. Zero-config (GitHub releases API).
+"""
+
+from __future__ import annotations
+
+from .base import Config, ConnectorError, http_get_json
+
+
+def recent_releases(target: str, cfg: Config, limit: int = 10) -> list[dict]:
+    if "/" not in target:
+        raise ConnectorError(f"changelog needs an owner/repo target, got {target!r}")
+    owner, repo = target.split("/", 1)
+    url = f"https://api.github.com/repos/{owner}/{repo}/releases?per_page={limit}"
+    headers = {"Accept": "application/vnd.github+json"}
+    if cfg.github_token:
+        headers["Authorization"] = f"Bearer {cfg.github_token}"
+    data = http_get_json(url, headers, cfg.timeout)
+    return parse_releases(data)
+
+
+def parse_releases(data: list) -> list[dict]:
+    out: list[dict] = []
+    for r in data or []:
+        out.append(
+            {
+                "name": r.get("name") or r.get("tag_name", ""),
+                "tag": r.get("tag_name", ""),
+                "published_at": r.get("published_at"),
+                "body": (r.get("body") or "")[:2000],
+                "url": r.get("html_url", ""),
+            }
+        )
+    return out

pmkit/connectors/github.py

@@ -0,0 +1,49 @@
+"""GitHub issues connector. Zero-config (better with GITHUB_TOKEN for rate limits)."""
+
+from __future__ import annotations
+
+from .base import Config, ConnectorError, candidate, http_get_json
+
+
+class GitHubConnector:
+    name = "github"
+
+    def available(self, cfg: Config) -> tuple[bool, str]:
+        return True, "public API (set GITHUB_TOKEN to raise rate limits)"
+
+    def fetch(self, target: str, cfg: Config, limit: int = 25) -> list[dict]:
+        if "/" not in target:
+            raise ConnectorError(f"github needs an owner/repo target, got {target!r}")
+        owner, repo = target.split("/", 1)
+        # sort=comments (desc) biases toward discussed issues; avoids the URL-encoding
+        # pitfalls of the reactions-+1 token while still surfacing high-signal pain.
+        url = (
+            f"https://api.github.com/repos/{owner}/{repo}/issues"
+            f"?state=open&sort=comments&direction=desc&per_page={min(limit, 100)}"
+        )
+        headers = {"Accept": "application/vnd.github+json"}
+        if cfg.github_token:
+            headers["Authorization"] = f"Bearer {cfg.github_token}"
+        data = http_get_json(url, headers, cfg.timeout)
+        return parse_issues(data, target)
+
+
+def parse_issues(data: list, target: str) -> list[dict]:
+    """Pure parser: GitHub issues JSON -> candidates. Skips pull requests."""
+    out: list[dict] = []
+    for it in data or []:
+        if "pull_request" in it:  # the issues endpoint also returns PRs
+            continue
+        reactions = (it.get("reactions") or {}).get("total_count", 0)
+        engagement = int(reactions) + int(it.get("comments", 0))
+        out.append(
+            candidate(
+                title=it.get("title", ""),
+                problem=(it.get("body") or "")[:1000],
+                source_type="github",
+                url=it.get("html_url", ""),
+                engagement=engagement,
+                created_at=it.get("created_at"),
+            )
+        )
+    return out

pmkit/connectors/hn.py

@@ -0,0 +1,42 @@
+"""Hacker News connector via the Algolia search API. Zero-config."""
+
+from __future__ import annotations
+
+import urllib.parse
+
+from .base import Config, candidate, http_get_json
+
+
+class HNConnector:
+    name = "hn"
+
+    def available(self, cfg: Config) -> tuple[bool, str]:
+        return True, "Algolia HN search (no key)"
+
+    def fetch(self, target: str, cfg: Config, limit: int = 25) -> list[dict]:
+        query = target.split("/")[-1]  # repo/project name
+        q = urllib.parse.quote(query)
+        url = f"https://hn.algolia.com/api/v1/search?query={q}&tags=story&hitsPerPage={limit}"
+        data = http_get_json(url, {}, cfg.timeout)
+        return parse_hn(data, query)
+
+
+def parse_hn(data: dict, query: str) -> list[dict]:
+    out: list[dict] = []
+    for hit in (data or {}).get("hits", []):
+        title = hit.get("title") or hit.get("story_title") or ""
+        if not title:
+            continue
+        object_id = hit.get("objectID")
+        url = hit.get("url") or f"https://news.ycombinator.com/item?id={object_id}"
+        out.append(
+            candidate(
+                title=title,
+                problem=f"Discussed on Hacker News re: {query}",
+                source_type="hn",
+                url=url,
+                engagement=int(hit.get("points", 0)) + int(hit.get("num_comments", 0)),
+                created_at=hit.get("created_at"),
+            )
+        )
+    return out

pmkit/connectors/reddit.py

@@ -0,0 +1,42 @@
+"""Reddit connector via the public search JSON. Keyless but rate-limited; degrades."""
+
+from __future__ import annotations
+
+import urllib.parse
+
+from .base import Config, candidate, http_get_json
+
+
+class RedditConnector:
+    name = "reddit"
+
+    def available(self, cfg: Config) -> tuple[bool, str]:
+        return True, "public search JSON (best-effort; may rate-limit)"
+
+    def fetch(self, target: str, cfg: Config, limit: int = 25) -> list[dict]:
+        query = target.split("/")[-1]
+        q = urllib.parse.quote(query)
+        url = f"https://www.reddit.com/search.json?q={q}&sort=top&t=year&limit={limit}"
+        data = http_get_json(url, {}, cfg.timeout)
+        return parse_reddit(data, query)
+
+
+def parse_reddit(data: dict, query: str) -> list[dict]:
+    out: list[dict] = []
+    for child in (data or {}).get("data", {}).get("children", []):
+        d = child.get("data", {})
+        title = d.get("title", "")
+        if not title:
+            continue
+        permalink = d.get("permalink", "")
+        out.append(
+            candidate(
+                title=title,
+                problem=(d.get("selftext") or f"Reddit discussion re: {query}")[:1000],
+                source_type="reddit",
+                url=f"https://www.reddit.com{permalink}" if permalink else d.get("url", ""),
+                engagement=int(d.get("score", 0)) + int(d.get("num_comments", 0)),
+                created_at=str(d.get("created_utc", "")),
+            )
+        )
+    return out

pmkit/connectors/web.py

@@ -0,0 +1,44 @@
+"""Web search connector via Brave Search. Requires BRAVE_API_KEY; skipped otherwise."""
+
+from __future__ import annotations
+
+import urllib.parse
+
+from .base import Config, ConnectorError, candidate, http_get_json
+
+
+class WebConnector:
+    name = "web"
+
+    def available(self, cfg: Config) -> tuple[bool, str]:
+        if cfg.brave_key:
+            return True, "Brave Search"
+        return False, "no BRAVE_API_KEY"
+
+    def fetch(self, target: str, cfg: Config, limit: int = 20) -> list[dict]:
+        if not cfg.brave_key:
+            raise ConnectorError("no BRAVE_API_KEY")
+        query = f"{target.split('/')[-1]} issues OR limitations OR feature request"
+        q = urllib.parse.quote(query)
+        url = f"https://api.search.brave.com/res/v1/web/search?q={q}&count={min(limit, 20)}"
+        headers = {"Accept": "application/json", "X-Subscription-Token": cfg.brave_key}
+        data = http_get_json(url, headers, cfg.timeout)
+        return parse_brave(data, target)
+
+
+def parse_brave(data: dict, target: str) -> list[dict]:
+    out: list[dict] = []
+    for r in (data or {}).get("web", {}).get("results", []):
+        title = r.get("title", "")
+        if not title:
+            continue
+        out.append(
+            candidate(
+                title=title,
+                problem=(r.get("description") or "")[:1000],
+                source_type="web",
+                url=r.get("url", ""),
+                engagement=0,  # web results carry no engagement signal
+            )
+        )
+    return out

pmkit/connectors/x.py

@@ -0,0 +1,50 @@
+"""X (Twitter) connector via API v2 recent search. Requires X_BEARER_TOKEN; skipped otherwise."""
+
+from __future__ import annotations
+
+import urllib.parse
+
+from .base import Config, ConnectorError, candidate, http_get_json
+
+
+class XConnector:
+    name = "x"
+
+    def available(self, cfg: Config) -> tuple[bool, str]:
+        if cfg.x_bearer:
+            return True, "X API v2 recent search"
+        return False, "no X_BEARER_TOKEN"
+
+    def fetch(self, target: str, cfg: Config, limit: int = 25) -> list[dict]:
+        if not cfg.x_bearer:
+            raise ConnectorError("no X_BEARER_TOKEN")
+        query = urllib.parse.quote(f"{target.split('/')[-1]} (bug OR feature OR wish) -is:retweet lang:en")
+        url = (
+            "https://api.twitter.com/2/tweets/search/recent"
+            f"?query={query}&max_results={min(max(limit, 10), 100)}"
+            "&tweet.fields=public_metrics,created_at"
+        )
+        headers = {"Authorization": f"Bearer {cfg.x_bearer}"}
+        data = http_get_json(url, headers, cfg.timeout)
+        return parse_x(data, target)
+
+
+def parse_x(data: dict, target: str) -> list[dict]:
+    out: list[dict] = []
+    for t in (data or {}).get("data", []):
+        text = t.get("text", "")
+        if not text:
+            continue
+        metrics = t.get("public_metrics", {})
+        engagement = int(metrics.get("like_count", 0)) + int(metrics.get("retweet_count", 0))
+        out.append(
+            candidate(
+                title=text[:120],
+                problem=text,
+                source_type="x",
+                url=f"https://twitter.com/i/web/status/{t.get('id')}",
+                engagement=engagement,
+                created_at=t.get("created_at"),
+            )
+        )
+    return out

pmkit/dedup.py

@@ -0,0 +1,64 @@
+"""Near-duplicate detection for discovered candidates.
+
+The backlog handles *exact* dedup via the normalized (target, dedup_key). This module adds
+*near*-duplicate matching so two differently-worded reports of the same problem on the same
+target collapse into one item (evidence accrues rather than duplicating). Pure functions,
+no I/O — fully unit-testable.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Optional
+
+_TOKEN_RE = re.compile(r"[a-z0-9]+")
+
+# A small stopword set keeps generic words from inflating similarity.
+_STOP = {
+    "the", "a", "an", "to", "of", "and", "or", "in", "on", "for", "is", "are",
+    "be", "with", "no", "not", "it", "this", "that", "when", "how", "i", "we",
+    "you", "my", "should", "would", "could", "can", "have", "has", "add", "support",
+}
+
+DEFAULT_THRESHOLD = 0.6
+
+
+def token_set(text: str) -> set[str]:
+    return {t for t in _TOKEN_RE.findall((text or "").lower()) if t not in _STOP and len(t) > 2}
+
+
+def jaccard(a: set[str], b: set[str]) -> float:
+    if not a and not b:
+        return 0.0
+    inter = len(a & b)
+    union = len(a | b)
+    return inter / union if union else 0.0
+
+
+def similarity(text_a: str, text_b: str) -> float:
+    """Token-set Jaccard over the combined title+problem text of two candidates."""
+    return jaccard(token_set(text_a), token_set(text_b))
+
+
+def _candidate_text(item: dict) -> str:
+    return f"{item.get('title', '')} {item.get('problem', '')}"
+
+
+def find_near_duplicate(
+    cand: dict,
+    existing: list[dict],
+    threshold: float = DEFAULT_THRESHOLD,
+) -> Optional[dict]:
+    """Return the most similar existing item above ``threshold``, or None.
+
+    ``cand`` and each ``existing`` item are dicts with 'title' and 'problem'.
+    """
+    cand_text = _candidate_text(cand)
+    best: Optional[dict] = None
+    best_score = threshold
+    for item in existing:
+        score = similarity(cand_text, _candidate_text(item))
+        if score >= best_score:
+            best = item
+            best_score = score
+    return best

pmkit/discover.py

@@ -0,0 +1,83 @@
+"""Discovery orchestration: run connectors, dedup, and write candidates to the backlog.
+
+Connectors are injectable so the orchestration is testable without the network. Each
+candidate is matched against the existing backlog (near-duplicate by similarity, exact by
+the backlog's own key) — a match attaches evidence, a miss creates a new item. Candidates
+with weak engagement or no source are flagged low-confidence rather than dropped (R3).
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from .backlog import Backlog, make_dedup_key
+from .connectors import get_connectors
+from .connectors.base import Config, ConnectorError
+from .dedup import DEFAULT_THRESHOLD, find_near_duplicate
+
+
+def run_discovery(
+    backlog: Backlog,
+    target: str,
+    connectors: Optional[list] = None,
+    cfg: Optional[Config] = None,
+    limit: int = 25,
+    near_threshold: float = DEFAULT_THRESHOLD,
+) -> dict:
+    cfg = cfg or Config.from_env()
+    connectors = connectors if connectors is not None else get_connectors()
+
+    summary = {
+        "target": target,
+        "fetched": 0,
+        "new": 0,
+        "merged": 0,
+        "low_confidence": 0,
+        "by_source": {},
+        "skipped": [],
+    }
+
+    # Seed with existing items for this target so re-runs dedup across runs (R10).
+    current = [it for it in backlog.list() if it["target"] == target]
+
+    for conn in connectors:
+        ok, reason = conn.available(cfg)
+        if not ok:
+            summary["skipped"].append({"source": conn.name, "reason": reason})
+            continue
+        try:
+            cands = conn.fetch(target, cfg, limit)
+        except ConnectorError as e:
+            summary["skipped"].append({"source": conn.name, "reason": str(e)})
+            continue
+        except Exception as e:  # defensive: a connector bug must not abort the run
+            summary["skipped"].append({"source": conn.name, "reason": f"unexpected: {e}"})
+            continue
+
+        summary["by_source"][conn.name] = len(cands)
+        for cand in cands:
+            summary["fetched"] += 1
+            low_conf = cand["engagement"] < cfg.min_engagement or not cand["source"].get("url")
+            # Match exact-key first (the same dedup add_candidate would do internally), then
+            # near-duplicate — so a candidate that add_candidate would merge is counted as
+            # 'merged', not 'new' (fixes the new-count overcount).
+            key = make_dedup_key(target, cand["problem"])
+            dup = backlog.find_existing(target, key) or find_near_duplicate(cand, current, near_threshold)
+            if dup is not None:
+                backlog.attach_evidence(dup["id"], [cand["source"]])
+                summary["merged"] += 1
+                continue
+            opp_id = backlog.add_candidate(
+                target=target,
+                title=cand["title"],
+                problem=cand["problem"],
+                sources=[cand["source"]],
+                low_confidence=low_conf,
+            )
+            summary["new"] += 1
+            if low_conf:
+                summary["low_confidence"] += 1
+            # Make this candidate visible to later ones in the same run.
+            current.append({"id": opp_id, "title": cand["title"], "problem": cand["problem"]})
+
+    return summary

pmkit/dogfood/__init__.py

@@ -0,0 +1,7 @@
+"""pm-dogfood deterministic helpers.
+
+The pm-dogfood *skill* owns inference and judgment (infer the usage scenario from a
+product's docs, decide what counts as a gap, confirm reproducibility). This package holds
+the mechanical, unit-testable pieces it calls: the clean-room install runner, the UI and
+MCP drivers, the report/parity builder, and the confirmed+deduped backlog filer.
+"""

pmkit/dogfood/file_gaps.py

@@ -0,0 +1,52 @@
+"""File confirmed dogfood gaps back to the opportunity backlog (deduped).
+
+Closes the funnel loop: a reproducible doc-vs-reality gap becomes a new opportunity.
+Only *confirmed* gaps are filed (the skill marks the ones that reproduced on re-run);
+flaky/unconfirmed gaps stay report-only. Filing reuses the backlog's exact-key dedup, so
+re-running pm-dogfood doesn't pile up duplicates. Each filed item carries a
+``source=dogfood`` provenance entry.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from ..backlog import Backlog, make_dedup_key
+from .report import Finding
+
+
+def file_gaps(
+    backlog: Backlog,
+    target: str,
+    gaps: Iterable[Finding],
+    confirmed: set[str],
+    *,
+    source: str = "dogfood",
+) -> dict:
+    """File confirmed gaps as backlog opportunities; skip unconfirmed; dedup the rest.
+
+    ``confirmed`` is the set of gap titles that reproduced. Returns counts of
+    filed / deduped / skipped (report-only) gap titles.
+    """
+    filed: list[int] = []
+    deduped: list[int] = []
+    skipped: list[str] = []
+    for g in gaps:
+        if g.title not in confirmed:
+            skipped.append(g.title)
+            continue
+        before = len(backlog.list())
+        opp_id = backlog.add_candidate(
+            target=target,
+            title=f"[dogfood] {g.title}",
+            problem=f"{g.claim} -> observed: {g.observed}".strip(),
+            # key on the gap's title (its real identity) so distinct gaps with similar
+            # output don't collapse into one — claim text is constant per interface.
+            dedup_key=make_dedup_key(target, f"{g.title} {g.observed}"),
+            sources=[{"type": source, "url": "", "excerpt": (g.observed or "")[:200]}],
+        )
+        if len(backlog.list()) > before:
+            filed.append(opp_id)
+        else:
+            deduped.append(opp_id)
+    return {"filed": filed, "deduped": deduped, "skipped": skipped}