researchforge 0.1.0__py3-none-any.whl

researchforge/__init__.py

@@ -0,0 +1,3 @@
+"""ResearchForge: from papers to proof."""
+
+__version__ = "0.1.0"

researchforge/__main__.py

@@ -0,0 +1,4 @@
+from researchforge.cli import app
+
+if __name__ == "__main__":
+    app()

researchforge/analytics/__init__.py

@@ -0,0 +1 @@
+"""Opt-in, local-only, privacy-preserving beta analytics (spec §20)."""

researchforge/analytics/cli.py

@@ -0,0 +1,93 @@
+"""`researchforge analytics` sub-app: opt-in local beta analytics."""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from researchforge.analytics.service import (
+    analytics_path,
+    compute_metrics,
+    is_enabled,
+    load_events,
+    set_enabled,
+)
+from researchforge.utils.output import JsonOption
+
+analytics_app = typer.Typer(
+    name="analytics", no_args_is_help=True, help="Opt-in, local-only beta analytics."
+)
+
+COLLECTION_NOTICE = """\
+Analytics are LOCAL-ONLY and nothing is ever transmitted anywhere.
+
+When enabled, coarse events append to .researchforge/analytics.jsonl:
+  - an event name (e.g. baseline_completed), a timestamp, ok/failed,
+    and a short failure category.
+Never collected: source code, paper text, secrets, datasets, metric
+values, or logs. The file is yours — share it with a beta report only
+if you choose to. Disable any time: researchforge analytics disable\
+"""
+
+
+@analytics_app.command("enable")
+def enable_command(json_output: JsonOption = False) -> None:
+    """Enable local analytics (prints exactly what is and is not collected)."""
+    set_enabled(True)
+    if json_output:
+        typer.echo(json.dumps({"analytics_enabled": True}))
+    else:
+        typer.echo(COLLECTION_NOTICE)
+        typer.echo("Enabled.")
+
+
+@analytics_app.command("disable")
+def disable_command(json_output: JsonOption = False) -> None:
+    """Disable local analytics (the existing log is kept; delete it if you wish)."""
+    set_enabled(False)
+    if json_output:
+        typer.echo(json.dumps({"analytics_enabled": False}))
+    else:
+        typer.echo(f"Disabled. Existing log (if any) remains at {analytics_path()}.")
+
+
+@analytics_app.command("status")
+def status_command(json_output: JsonOption = False) -> None:
+    """Show whether analytics are enabled and how many events are recorded."""
+    enabled = is_enabled()
+    count = len(load_events())
+    if json_output:
+        typer.echo(json.dumps({"analytics_enabled": enabled, "events_recorded": count}))
+    else:
+        typer.echo(f"Analytics: {'enabled' if enabled else 'disabled'} ({count} event(s) recorded)")
+
+
+@analytics_app.command("show")
+def show_command(json_output: JsonOption = False) -> None:
+    """Compute the beta metrics locally from the recorded events."""
+    metrics = compute_metrics()
+    if json_output:
+        typer.echo(metrics.model_dump_json(indent=2))
+        return
+    typer.echo(f"Events recorded: {metrics.events_recorded}")
+    for label, value in (
+        ("Time to first landscape", metrics.time_to_first_landscape_s),
+        ("Time to baseline", metrics.time_to_baseline_s),
+    ):
+        typer.echo(f"{label}: {f'{value:.0f}s' if value is not None else '—'}")
+    for label, rate in (
+        ("Baseline success rate", metrics.baseline_success_rate),
+        ("Experiment completion rate", metrics.experiment_completion_rate),
+        ("Valid-metrics rate", metrics.valid_metrics_rate),
+    ):
+        typer.echo(f"{label}: {f'{rate:.0%}' if rate is not None else '—'}")
+    typer.echo(
+        f"Validated findings: {metrics.validated_findings}; "
+        f"branches: {metrics.branches_created}; reports: {metrics.reports_generated}; "
+        f"packages: {metrics.packages_generated}"
+    )
+    if metrics.failure_categories:
+        typer.echo("Failure categories:")
+        for category, count in sorted(metrics.failure_categories.items()):
+            typer.echo(f"  {category}: {count}")

researchforge/analytics/service.py

@@ -0,0 +1,167 @@
+"""Local-only beta analytics (spec §20).
+
+Disabled by default. When enabled, coarse events append to
+`.researchforge/analytics.jsonl` — event name, timestamp, ok flag, and an
+optional short category. Never code, paper text, secrets, metric values, or
+logs; nothing is ever transmitted anywhere. The file exists so beta users
+can *choose* to share it, and so `researchforge analytics show` can compute
+the beta metrics locally. Recording must never break a command: failures to
+write are swallowed.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from researchforge.config.paths import config_path, researchforge_dir
+from researchforge.config.settings import load_settings
+
+ANALYTICS_FILENAME = "analytics.jsonl"
+
+# The spec §20 event vocabulary; record_event refuses anything else so the
+# log can never accumulate ad-hoc payloads.
+EVENTS = frozenset(
+    {
+        "initialized",
+        "doctor_passed",
+        "project_created",
+        "repo_scanned",
+        "papers_retrieved",
+        "landscape_imported",
+        "hypotheses_imported",
+        "contract_approved",
+        "baseline_completed",
+        "experiment_started",
+        "experiment_completed",
+        "validated_finding",
+        "branch_created",
+        "draft_pr_created",
+        "report_generated",
+        "package_generated",
+    }
+)
+
+
+def analytics_path(base: Path | None = None) -> Path:
+    return researchforge_dir(base) / ANALYTICS_FILENAME
+
+
+def is_enabled(base: Path | None = None) -> bool:
+    try:
+        return load_settings(base).analytics_enabled
+    except (OSError, ValueError):
+        return False
+
+
+def set_enabled(enabled: bool, base: Path | None = None) -> None:
+    """Flip the opt-in flag in `.researchforge/config.json`, preserving other keys."""
+    path = config_path(base)
+    raw: dict[str, object] = {}
+    if path.is_file():
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    raw["analytics_enabled"] = enabled
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(raw, indent=2) + "\n", encoding="utf-8")
+
+
+def record_event(
+    name: str, ok: bool = True, category: str | None = None, base: Path | None = None
+) -> None:
+    """Append one coarse event if analytics are enabled; never raise."""
+    assert name in EVENTS, f"unknown analytics event: {name}"
+    try:
+        if not is_enabled(base):
+            return
+        entry: dict[str, object] = {
+            "event": name,
+            "ts": datetime.now(UTC).isoformat(timespec="seconds"),
+            "ok": ok,
+        }
+        if category is not None:
+            entry["category"] = category
+        with analytics_path(base).open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(entry) + "\n")
+    except OSError:
+        return
+
+
+def load_events(base: Path | None = None) -> list[dict[str, object]]:
+    path = analytics_path(base)
+    if not path.is_file():
+        return []
+    events = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if line.strip():
+            events.append(json.loads(line))
+    return events
+
+
+class BetaMetrics(BaseModel):
+    """Spec §20 key beta metrics, computed locally from the event log."""
+
+    events_recorded: int
+    time_to_first_landscape_s: float | None = None
+    time_to_baseline_s: float | None = None
+    baseline_success_rate: float | None = None
+    experiment_completion_rate: float | None = None
+    valid_metrics_rate: float | None = None
+    validated_findings: int = 0
+    branches_created: int = 0
+    reports_generated: int = 0
+    packages_generated: int = 0
+    failure_categories: dict[str, int] = {}
+
+
+def _first_ts(events: list[dict[str, object]], name: str, ok_only: bool = False) -> datetime | None:
+    for event in events:
+        if event.get("event") == name and (not ok_only or event.get("ok")):
+            return datetime.fromisoformat(str(event["ts"]))
+    return None
+
+
+def _rate(events: list[dict[str, object]], name: str) -> float | None:
+    matching = [e for e in events if e.get("event") == name]
+    if not matching:
+        return None
+    return sum(1 for e in matching if e.get("ok")) / len(matching)
+
+
+def compute_metrics(base: Path | None = None) -> BetaMetrics:
+    events = load_events(base)
+    created = _first_ts(events, "project_created")
+    landscape = _first_ts(events, "landscape_imported", ok_only=True)
+    baseline = _first_ts(events, "baseline_completed", ok_only=True)
+
+    started = sum(1 for e in events if e.get("event") == "experiment_started")
+    completed = [e for e in events if e.get("event") == "experiment_completed"]
+
+    failure_categories = Counter(
+        str(e.get("category", "uncategorized"))
+        for e in events
+        if not e.get("ok") and e.get("event") in ("baseline_completed", "experiment_completed")
+    )
+
+    return BetaMetrics(
+        events_recorded=len(events),
+        time_to_first_landscape_s=(
+            (landscape - created).total_seconds() if created and landscape else None
+        ),
+        time_to_baseline_s=((baseline - created).total_seconds() if created and baseline else None),
+        baseline_success_rate=_rate(events, "baseline_completed"),
+        experiment_completion_rate=(
+            sum(1 for e in completed if e.get("ok")) / started if started else None
+        ),
+        valid_metrics_rate=_rate(events, "experiment_completed"),
+        validated_findings=sum(
+            1 for e in events if e.get("event") == "validated_finding" and e.get("ok")
+        ),
+        branches_created=sum(1 for e in events if e.get("event") == "branch_created"),
+        reports_generated=sum(1 for e in events if e.get("event") == "report_generated"),
+        packages_generated=sum(1 for e in events if e.get("event") == "package_generated"),
+        failure_categories=dict(failure_categories),
+    )

researchforge/claude/__init__.py

@@ -0,0 +1 @@
+"""Claude Code integration: packaged project skills and their installer."""

researchforge/claude/cli.py

@@ -0,0 +1,87 @@
+"""`researchforge claude` sub-app: manage the project-level Claude skills."""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from researchforge.claude.installer import (
+    InstallReport,
+    SkillAction,
+    install_skills,
+    skills_status,
+    uninstall_skills,
+)
+from researchforge.utils.output import JsonOption
+
+claude_app = typer.Typer(name="claude", no_args_is_help=True, help="Claude Code skills.")
+
+ForceOption = typer.Option(
+    False,
+    "--force",
+    help="Overwrite/remove skills even if they were modified after installation.",
+)
+
+UserOption = typer.Option(
+    False,
+    "--user",
+    help="Target ~/.claude/skills/ (every project on this machine) instead of this repository.",
+)
+
+_ACTION_MARKERS = {
+    SkillAction.INSTALLED: "+",
+    SkillAction.UPDATED: "^",
+    SkillAction.UNCHANGED: "=",
+    SkillAction.SKIPPED_MODIFIED: "!",
+    SkillAction.REMOVED: "-",
+    SkillAction.LEFT_MODIFIED: "!",
+    SkillAction.MISSING: "?",
+    SkillAction.MODIFIED: "!",
+}
+
+
+def _echo_report(report: InstallReport, json_output: bool) -> None:
+    if json_output:
+        typer.echo(json.dumps(report.model_dump(), indent=2))
+        return
+    for result in report.results:
+        typer.echo(f"{_ACTION_MARKERS[result.action]} {result.skill}: {result.action.value}")
+
+
+@claude_app.command("install")
+def install_command(
+    force: bool = ForceOption, user: bool = UserOption, json_output: JsonOption = False
+) -> None:
+    """Install the ResearchForge skills into this repository's .claude/skills/."""
+    report = install_skills(force=force, user=user)
+    _echo_report(report, json_output)
+    if not json_output:
+        if report.conflicts:
+            typer.echo(
+                "Some skills were modified after installation and were left untouched; "
+                "re-run with --force to overwrite them."
+            )
+        typer.echo(f"Skills directory: {report.skills_dir}")
+
+
+@claude_app.command("uninstall")
+def uninstall_command(
+    force: bool = ForceOption, user: bool = UserOption, json_output: JsonOption = False
+) -> None:
+    """Remove the installed ResearchForge skills (user-modified files are kept)."""
+    report = uninstall_skills(force=force, user=user)
+    _echo_report(report, json_output)
+    if not json_output:
+        left = [r for r in report.results if r.action is SkillAction.LEFT_MODIFIED]
+        if left:
+            typer.echo(
+                "Modified skills were left in place; re-run with --force to remove them too."
+            )
+
+
+@claude_app.command("status")
+def status_command(user: bool = UserOption, json_output: JsonOption = False) -> None:
+    """Show whether each packaged skill is installed, modified, or missing."""
+    report = skills_status(user=user)
+    _echo_report(report, json_output)

researchforge/claude/installer.py

@@ -0,0 +1,184 @@
+"""Install packaged Claude skills into a repository's `.claude/skills/`.
+
+The skills are UX, not a security boundary — every gate they describe is
+enforced by the Python engine regardless of what a skill (or Claude) says.
+Installation is manifest-based so ResearchForge never overwrites or removes
+Claude configuration it does not own: a sha256 per installed file is recorded
+in `.researchforge/claude-skills-manifest.json`, and any file that no longer
+matches its recorded hash is treated as user-owned.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from enum import StrEnum
+from importlib import resources
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from researchforge.config.paths import researchforge_dir
+
+SKILLS_PACKAGE = "researchforge.claude.skills"
+MANIFEST_FILENAME = "claude-skills-manifest.json"
+CLAUDE_SKILLS_DIR = Path(".claude") / "skills"
+
+
+class SkillAction(StrEnum):
+    INSTALLED = "installed"
+    UPDATED = "updated"
+    UNCHANGED = "unchanged"
+    SKIPPED_MODIFIED = "skipped_modified"
+    REMOVED = "removed"
+    LEFT_MODIFIED = "left_modified"
+    MISSING = "missing"
+    MODIFIED = "modified"
+
+
+class SkillReport(BaseModel):
+    skill: str
+    action: SkillAction
+    path: str
+
+
+class InstallReport(BaseModel):
+    skills_dir: str
+    results: list[SkillReport]
+
+    @property
+    def conflicts(self) -> list[SkillReport]:
+        return [r for r in self.results if r.action is SkillAction.SKIPPED_MODIFIED]
+
+
+class SkillsManifest(BaseModel):
+    """sha256 of each installed SKILL.md, keyed by skill name."""
+
+    version: int = 1
+    hashes: dict[str, str] = {}
+
+
+def _sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def manifest_path(base: Path | None = None, user: bool = False) -> Path:
+    if user:
+        # User-level installs have no project `.researchforge/`; keep the
+        # manifest next to the user's Claude config instead.
+        return Path.home() / ".claude" / f"researchforge-{MANIFEST_FILENAME}"
+    return researchforge_dir(base) / MANIFEST_FILENAME
+
+
+def _skills_root(base: Path | None, user: bool) -> Path:
+    if user:
+        return Path.home() / ".claude" / "skills"
+    return (base if base is not None else Path.cwd()) / CLAUDE_SKILLS_DIR
+
+
+def load_manifest(base: Path | None = None, user: bool = False) -> SkillsManifest:
+    path = manifest_path(base, user)
+    if not path.is_file():
+        return SkillsManifest()
+    return SkillsManifest.model_validate_json(path.read_text(encoding="utf-8"))
+
+
+def save_manifest(manifest: SkillsManifest, base: Path | None = None, user: bool = False) -> None:
+    path = manifest_path(base, user)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(manifest.model_dump_json(indent=2) + "\n", encoding="utf-8")
+
+
+def list_packaged_skills() -> dict[str, str]:
+    """Skill name -> SKILL.md content, from the wheel's packaged assets."""
+    skills: dict[str, str] = {}
+    root = resources.files(SKILLS_PACKAGE)
+    for entry in root.iterdir():
+        if entry.is_dir():
+            skill_file = entry / "SKILL.md"
+            if skill_file.is_file():
+                skills[entry.name] = skill_file.read_text(encoding="utf-8")
+    return dict(sorted(skills.items()))
+
+
+def install_skills(
+    base: Path | None = None, force: bool = False, user: bool = False
+) -> InstallReport:
+    """Copy packaged skills into `.claude/skills/`, never clobbering user edits.
+
+    `user=True` installs into `~/.claude/skills/` (available in every project
+    and Claude account on this machine) instead of the repository.
+
+    Per skill: missing -> write; unchanged since our last install (manifest
+    hash matches) -> update in place; anything else -> skip unless `force`.
+    """
+    skills_root = _skills_root(base, user)
+    manifest = load_manifest(base, user)
+    results: list[SkillReport] = []
+
+    for name, content in list_packaged_skills().items():
+        target = skills_root / name / "SKILL.md"
+        packaged_hash = _sha256(content.encode("utf-8"))
+        if target.is_file():
+            current_hash = _sha256(target.read_bytes())
+            if current_hash == packaged_hash:
+                action = SkillAction.UNCHANGED
+            elif current_hash == manifest.hashes.get(name) or force:
+                action = SkillAction.UPDATED
+            else:
+                action = SkillAction.SKIPPED_MODIFIED
+        else:
+            action = SkillAction.INSTALLED
+
+        if action in (SkillAction.INSTALLED, SkillAction.UPDATED):
+            target.parent.mkdir(parents=True, exist_ok=True)
+            target.write_text(content, encoding="utf-8")
+        if action is not SkillAction.SKIPPED_MODIFIED:
+            manifest.hashes[name] = packaged_hash
+        results.append(SkillReport(skill=name, action=action, path=str(target)))
+
+    save_manifest(manifest, base, user)
+    return InstallReport(skills_dir=str(skills_root), results=results)
+
+
+def uninstall_skills(
+    base: Path | None = None, force: bool = False, user: bool = False
+) -> InstallReport:
+    """Remove installed skills; user-modified files are left unless `force`."""
+    skills_root = _skills_root(base, user)
+    manifest = load_manifest(base, user)
+    results: list[SkillReport] = []
+
+    for name in list_packaged_skills():
+        target = skills_root / name / "SKILL.md"
+        recorded = manifest.hashes.get(name)
+        if not target.is_file():
+            action = SkillAction.MISSING
+        elif _sha256(target.read_bytes()) == recorded or force:
+            target.unlink()
+            if not any(target.parent.iterdir()):
+                target.parent.rmdir()
+            action = SkillAction.REMOVED
+        else:
+            action = SkillAction.LEFT_MODIFIED
+        if action is not SkillAction.LEFT_MODIFIED:
+            manifest.hashes.pop(name, None)
+        results.append(SkillReport(skill=name, action=action, path=str(target)))
+
+    save_manifest(manifest, base, user)
+    return InstallReport(skills_dir=str(skills_root), results=results)
+
+
+def skills_status(base: Path | None = None, user: bool = False) -> InstallReport:
+    """Per-skill state: unchanged (as packaged), modified, or missing."""
+    skills_root = _skills_root(base, user)
+    results: list[SkillReport] = []
+    for name, content in list_packaged_skills().items():
+        target = skills_root / name / "SKILL.md"
+        if not target.is_file():
+            action = SkillAction.MISSING
+        elif _sha256(target.read_bytes()) == _sha256(content.encode("utf-8")):
+            action = SkillAction.UNCHANGED
+        else:
+            action = SkillAction.MODIFIED
+        results.append(SkillReport(skill=name, action=action, path=str(target)))
+    return InstallReport(skills_dir=str(skills_root), results=results)

researchforge/claude/skills/researchforge-baseline/SKILL.md

@@ -0,0 +1,65 @@
+---
+name: researchforge-baseline
+description: Set up the experiment contract and run the frozen baseline benchmark. Use when an improve-repository project needs its evaluation defined, the contract approved, or the baseline measured.
+---
+
+# Contract and baseline
+
+The contract (`researchforge.yaml` at the repository root) freezes the
+evaluation: primary metric + direction, hard constraints, editable vs
+protected paths, commands, resource limits, network mode, shipping flags.
+Approval hashes it; later edits are detected as drift and refused until
+re-approved. **The contract is the boundary — experiments can only change
+`editable_paths` and are judged only by the contracted benchmark.**
+
+## 1. Draft the contract
+
+```bash
+researchforge contract generate --json
+```
+
+Then edit `researchforge.yaml` **with the user** — these are their choices,
+not yours: the benchmark command (`execution.full_command`) must print or
+write a machine-readable result (`execution.result_file`), the primary
+metric must match a key in that result, and `permissions.editable_paths` /
+`protected_paths` decide what experiments may touch. Benchmarks and
+evaluation code belong in `protected_paths`.
+
+## 2. Validate and approve
+
+```bash
+researchforge contract validate --json
+```
+
+Fix any of the semantic errors it reports (they are field-level and
+actionable). Then show the user a summary of what they are approving —
+metric, constraints, commands, limits, paths — and ask them to run:
+
+```bash
+researchforge contract approve
+```
+
+The approval is a typed confirmation. Only run it with `--yes` yourself if
+the user has explicitly said in this conversation to approve this contract.
+
+## 3. Run the baseline
+
+```bash
+researchforge baseline run --json
+researchforge baseline show --json
+```
+
+The baseline runs in an isolated detached worktree; the user's checkout is
+never touched. Report the recorded baseline metrics from the JSON. All
+future experiments are compared against this frozen measurement.
+
+Next: the researchforge-plan skill designs experiments against a hypothesis.
+
+## Rules
+
+- The Python engine is the boundary: never work around a validation error, a
+  protected path, or an approval gate — fix the artifact or ask the user.
+- Approvals belong to the user: never pass `--yes` or type a confirmation
+  unless the user explicitly approved that step in this conversation.
+- Ground every summary in stored data: quote only numbers returned by
+  `--json` output or files under `.researchforge/` — never invent metrics.

researchforge/claude/skills/researchforge-doctor/SKILL.md

@@ -0,0 +1,33 @@
+---
+name: researchforge-doctor
+description: Check that ResearchForge's dependencies (git, Python, optionally Docker) are available and explain any failures. Use when setup fails, before starting a project, or when the user asks whether their machine is ready.
+---
+
+# Diagnose the ResearchForge environment
+
+```bash
+researchforge doctor --json
+```
+
+Each entry has `name`, `ok`, `required`, `detail`, and an optional `hint`.
+
+- All required checks pass → say the environment is ready and continue.
+- A required check fails → explain the failure in plain language, show the
+  `hint`, and stop: later commands will refuse to run. Do not try to work
+  around a failed required check.
+- An optional check fails (e.g. Docker) → note the limitation: experiments
+  fall back to `.venv` mode, which is dependency isolation, not a security
+  sandbox — only trusted repositories should be run there.
+
+If skills seem out of date or missing, `researchforge claude status --json`
+reports the installed/modified/missing state of each skill and
+`researchforge claude install` refreshes unmodified ones.
+
+## Rules
+
+- The Python engine is the boundary: never work around a validation error, a
+  protected path, or an approval gate — fix the artifact or ask the user.
+- Approvals belong to the user: never pass `--yes` or type a confirmation
+  unless the user explicitly approved that step in this conversation.
+- Ground every summary in stored data: quote only numbers returned by
+  `--json` output or files under `.researchforge/` — never invent metrics.