ultimate-pi 0.3.1 → 0.4.0

package/.pi/scripts/harness_web/scrape.py

@@ -0,0 +1,120 @@
+"""Scrape and map URLs via Scrapling fetchers."""
+
+from __future__ import annotations
+
+import time
+from urllib.parse import urljoin, urlparse
+
+from scrapling.fetchers import Fetcher, StealthyFetcher, StealthySession
+
+from .config import HarnessWebConfig
+from .output import write_page_markdown
+
+
+def _fetch_kwargs(config: HarnessWebConfig, *, wait_ms: int | None = None) -> dict:
+    kw: dict = {"timeout": config.timeout_ms}
+    if config.proxy:
+        kw["proxy"] = config.proxy
+    if wait_ms is not None:
+        kw["wait"] = wait_ms
+    return kw
+
+
+def fetch_page(url: str, *, config: HarnessWebConfig, fast: bool, wait_ms: int | None):
+    kw = _fetch_kwargs(config, wait_ms=wait_ms)
+    if fast:
+        return Fetcher.get(url, timeout=config.timeout_sec, proxy=config.proxy)
+    return StealthyFetcher.fetch(url, **kw)
+
+
+def scrape_url(
+    url: str,
+    output: str,
+    *,
+    config: HarnessWebConfig,
+    fast: bool,
+    wait_ms: int | None,
+) -> None:
+    from pathlib import Path
+
+    page = fetch_page(url, config=config, fast=fast, wait_ms=wait_ms)
+    write_page_markdown(Path(output), page, main_content_only=True)
+
+
+def map_url(
+    url: str,
+    output: str,
+    *,
+    config: HarnessWebConfig,
+    fast: bool,
+    limit: int,
+) -> None:
+    import json
+    from pathlib import Path
+
+    page = fetch_page(url, config=config, fast=fast, wait_ms=None)
+    base = urlparse(url)
+    host = (base.hostname or "").lower()
+    links: list[dict[str, str]] = []
+    seen: set[str] = set()
+
+    for el in page.css("a[href]"):
+        if len(links) >= limit:
+            break
+        href = (el.attrib.get("href") or "").strip()
+        if not href or href.startswith("#") or href.lower().startswith("javascript:"):
+            continue
+        absolute = urljoin(url, href)
+        parsed = urlparse(absolute)
+        if (parsed.hostname or "").lower() != host:
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        title = (el.get_all_text(strip=True) or "").strip()
+        links.append({"url": absolute, "title": title})
+
+    out = Path(output)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps({"url": url, "links": links}, indent=2) + "\n", encoding="utf-8")
+
+
+def bulk_scrape(
+    urls: list[str],
+    out_dir: str,
+    *,
+    config: HarnessWebConfig,
+    fast: bool,
+    sleep_sec: float,
+) -> list[str]:
+    from pathlib import Path
+
+    root = Path(out_dir)
+    root.mkdir(parents=True, exist_ok=True)
+    failures: list[str] = []
+
+    if fast:
+        for i, url in enumerate(urls):
+            if i and sleep_sec > 0:
+                time.sleep(sleep_sec)
+            safe = urlparse(url).netloc.replace(".", "_") + ".md"
+            dest = root / safe
+            try:
+                scrape_url(url, str(dest), config=config, fast=True, wait_ms=None)
+            except Exception as err:  # noqa: BLE001
+                failures.append(f"{url}: {err}")
+        return failures
+
+    kw = _fetch_kwargs(config)
+    with StealthySession(headless=True, **kw) as session:
+        for i, url in enumerate(urls):
+            if i and sleep_sec > 0:
+                time.sleep(sleep_sec)
+            safe = urlparse(url).netloc.replace(".", "_") + ".md"
+            dest = root / safe
+            try:
+                page = session.fetch(url)
+                write_page_markdown(dest, page, main_content_only=True)
+            except Exception as err:  # noqa: BLE001
+                failures.append(f"{url}: {err}")
+    return failures

package/.pi/scripts/harness_web/search_ddg.py

@@ -0,0 +1,106 @@
+"""DuckDuckGo HTML SERP search via HTTP Fetcher."""
+
+from __future__ import annotations
+
+from urllib.parse import parse_qs, unquote, urlparse
+
+from scrapling.fetchers import Fetcher, StealthyFetcher
+
+from .config import HarnessWebConfig
+
+DDG_HTML_URL = "https://html.duckduckgo.com/html/"
+CHALLENGE_MARKERS = (
+    "anomaly-modal",
+    "bots use duckduckgo",
+    "please complete the following challenge",
+    "duckduckgo.com/y.js",
+)
+
+
+def _unwrap_ddg_href(href: str) -> str:
+    if not href:
+        return ""
+    if href.startswith("//"):
+        href = "https:" + href
+    parsed = urlparse(href)
+    if "duckduckgo.com" in (parsed.hostname or "") and parsed.path.startswith("/l/"):
+        qs = parse_qs(parsed.query)
+        if "uddg" in qs and qs["uddg"]:
+            return unquote(qs["uddg"][0])
+    return href
+
+
+def _looks_like_challenge(html: str) -> bool:
+    lower = html.lower()
+    return any(m in lower for m in CHALLENGE_MARKERS)
+
+
+def _parse_serp(page: Any, limit: int) -> list[dict[str, str]]:
+    results: list[dict[str, str]] = []
+    for block in page.css(".result"):
+        if len(results) >= limit:
+            break
+        links = block.css(".result__a")
+        if not links:
+            continue
+        link = links[0]
+        href = _unwrap_ddg_href(link.attrib.get("href", ""))
+        if not href.startswith("http"):
+            continue
+        title = (link.text or "").strip()
+        snippet_el = block.css(".result__snippet")
+        description = ""
+        if snippet_el:
+            description = (snippet_el[0].get_all_text(strip=True) or "").strip()
+        results.append({"url": href, "title": title, "description": description})
+    return results
+
+
+def search_ddg(
+    query: str,
+    *,
+    limit: int,
+    config: HarnessWebConfig,
+    impersonate: bool = True,
+) -> list[dict[str, str]]:
+    if config.search_engine != "ddg_html":
+        raise SystemExit(
+            f"Unsupported HARNESS_WEB_SEARCH_ENGINE={config.search_engine!r} (only ddg_html)"
+        )
+
+    kwargs: dict = {
+        "params": {"q": query},
+        "timeout": config.timeout_sec,
+    }
+    if config.proxy:
+        kwargs["proxy"] = config.proxy
+    if impersonate:
+        kwargs["impersonate"] = "chrome"
+
+    page = Fetcher.get(DDG_HTML_URL, **kwargs)
+    status = getattr(page, "status", 200)
+    html = getattr(page, "html_content", "") or ""
+
+    if status == 403 or _looks_like_challenge(html):
+        page = StealthyFetcher.fetch(
+            DDG_HTML_URL,
+            params={"q": query},
+            timeout=config.timeout_ms,
+            proxy=config.proxy,
+            headless=True,
+        )
+        html = getattr(page, "html_content", "") or ""
+        if _looks_like_challenge(html):
+            raise SystemExit(
+                "Search engine blocked (403/challenge page). "
+                "Try later, set HARNESS_WEB_PROXY, or reduce query rate."
+            )
+
+    parsed = _parse_serp(page, limit)
+    if not parsed and html:
+        # Empty selector set — likely HTML shape change
+        if "result__a" not in html:
+            raise SystemExit(
+                "Could not parse DuckDuckGo HTML results (page layout may have changed)."
+            )
+    return parsed

package/.pi/scripts/release.sh

@@ -0,0 +1,338 @@
+#!/usr/bin/env bash
+#
+# release.sh — Version bump, changelog, tag, and push
+# Usage: ./.pi/scripts/release.sh [patch|minor|major] [--dry-run]
+#
+set -euo pipefail
+
+# ─── Helpers ──────────────────────────────────────────────────────────────────
+warn()  { echo "⚠ $*" >&2; }
+abort() { echo "✗ $*" >&2; exit 1; }
+ok()    { echo "✓ $*"; }
+
+# ─── Step 0 — Parse arguments ─────────────────────────────────────────────────
+BUMP_TYPE=""
+DRY_RUN=false
+
+for arg in "$@"; do
+  case "$arg" in
+    patch|minor|major) BUMP_TYPE="$arg" ;;
+    --dry-run)         DRY_RUN=true ;;
+    *)                 abort "Unknown argument: $arg" ;;
+  esac
+done
+
+# ─── Step 1 — Infer bump type from commits if not provided ────────────────────
+LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
+
+if [ -z "$BUMP_TYPE" ]; then
+  ok "No bump type provided. Scanning commits since last tag…"
+
+  if [ -z "$LAST_TAG" ]; then
+    COMMIT_LOG=$(git log --format="%s" HEAD 2>/dev/null || true)
+  else
+    COMMIT_LOG=$(git log --format="%s" "${LAST_TAG}..HEAD" 2>/dev/null || true)
+  fi
+
+  if [ -z "$COMMIT_LOG" ]; then
+    abort "No commits since last tag. Nothing to release."
+  fi
+
+  # Inference rules
+  if echo "$COMMIT_LOG" | grep -qE '^feat!:|BREAKING CHANGE'; then
+    BUMP_TYPE="major"
+  elif echo "$COMMIT_LOG" | grep -qE '^feat:'; then
+    BUMP_TYPE="minor"
+  else
+    BUMP_TYPE="patch"
+  fi
+
+  ok "Inferred bump type: $BUMP_TYPE"
+fi
+
+# ─── Step 2 — Read current version and validate semver ────────────────────────
+CURRENT_VERSION=$(node -e "console.log(require('./package.json').version)" 2>/dev/null) \
+  || abort "Failed to read version from package.json"
+
+if ! [[ "$CURRENT_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  abort "Invalid semver in package.json: $CURRENT_VERSION"
+fi
+
+NEW_VERSION=$(node -e "
+const [maj, min, pat] = '$CURRENT_VERSION'.split('.').map(Number);
+const bump = '$BUMP_TYPE';
+if (bump === 'major') console.log((maj + 1) + '.0.0');
+else if (bump === 'minor') console.log(maj + '.' + (min + 1) + '.0');
+else console.log(maj + '.' + min + '.' + (pat + 1));
+")
+
+ok "Version: $CURRENT_VERSION → $NEW_VERSION"
+
+# ─── Step 3 — Pre-flight checks ───────────────────────────────────────────────
+
+# Must be in a git repo
+git rev-parse --is-inside-work-tree >/dev/null 2>&1 \
+  || abort "Not a git repository."
+
+# Must have origin remote
+git remote -v | grep -q origin \
+  || abort "No 'origin' remote configured."
+
+# Must be on a branch (not detached HEAD)
+BRANCH=$(git symbolic-ref -q HEAD 2>/dev/null | sed 's|^refs/heads/||') \
+  || abort "Detached HEAD. Switch to a branch first."
+
+# Must have clean working tree (warn only in dry-run)
+git diff --quiet && git diff --cached --quiet
+if [ $? -ne 0 ]; then
+  if [ "$DRY_RUN" = true ]; then
+    warn "Working tree is dirty — actual release would be blocked."
+  else
+    abort "Working tree is dirty. Commit or stash changes first."
+  fi
+fi
+
+# No duplicate tag locally or on remote
+if git rev-parse "v$NEW_VERSION" >/dev/null 2>&1; then
+  abort "Tag v$NEW_VERSION already exists locally."
+fi
+if git ls-remote --tags origin "refs/tags/v$NEW_VERSION" >/dev/null 2>&1; then
+  abort "Tag v$NEW_VERSION already exists on remote."
+fi
+
+# ─── Step 4 — Gather commits since last tag ───────────────────────────────────
+if [ -z "$LAST_TAG" ]; then
+  COMMITS=$(git log --oneline --no-merges HEAD)
+else
+  COMMITS=$(git log --oneline --no-merges "${LAST_TAG}..HEAD")
+fi
+
+COMMIT_COUNT=$(echo "$COMMITS" | grep -c '^' || echo 0)
+
+if [ "$COMMIT_COUNT" -eq 0 ]; then
+  abort "No commits since last tag. Nothing to release."
+fi
+
+# ─── Step 5 — Generate changelog entry ────────────────────────────────────────
+
+# Map conventional commit prefix → section
+map_prefix() {
+  local msg="$1"
+  case "$msg" in
+    feat!:*|*"BREAKING CHANGE"*) echo "breaking" ;;
+    feat:*)                      echo "features" ;;
+    fix:*)                      echo "fixes" ;;
+    perf:*)                     echo "perf" ;;
+    refactor:*)                 echo "refactor" ;;
+    docs:*)                     echo "docs" ;;
+    style:*)                    echo "style" ;;
+    test:*)                     echo "tests" ;;
+    chore:*)                    echo "chores" ;;
+    ci:*)                       echo "ci" ;;
+    build:*)                    echo "build" ;;
+    *)                          echo "chores" ;;
+  esac
+}
+
+declare -A SECTIONS=(
+  [breaking]="⚠️ Breaking Changes"
+  [features]="✨ Features"
+  [fixes]="🐛 Fixes"
+  [perf]="⚡ Performance"
+  [refactor]="♻️ Refactoring"
+  [docs]="📖 Documentation"
+  [style]="🎨 Style"
+  [tests]="✅ Tests"
+  [chores]="🔧 Chores"
+  [ci]="🔄 CI/CD"
+  [build]="📦 Build"
+)
+
+# Build per-section entries
+entries_breaking=""
+entries_features=""
+entries_fixes=""
+entries_perf=""
+entries_refactor=""
+entries_docs=""
+entries_style=""
+entries_tests=""
+entries_chores=""
+entries_ci=""
+entries_build=""
+
+while IFS= read -r line; do
+  [ -z "$line" ] && continue
+  # Strip the short sha prefix (first word)
+  msg="${line#* }"
+  # Strip conventional commit prefix for display
+  display="$msg"
+  display=$(echo "$display" | sed -E 's/^[a-z]+(\([a-z0-9_-]+\))?!?:\s*//')
+  prefix=$(map_prefix "$msg")
+  case "$prefix" in
+    breaking) entries_breaking="${entries_breaking}- $display
+" ;;
+    features) entries_features="${entries_features}- $display
+" ;;
+    fixes)    entries_fixes="${entries_fixes}- $display
+" ;;
+    perf)     entries_perf="${entries_perf}- $display
+" ;;
+    refactor) entries_refactor="${entries_refactor}- $display
+" ;;
+    docs)     entries_docs="${entries_docs}- $display
+" ;;
+    style)    entries_style="${entries_style}- $display
+" ;;
+    tests)    entries_tests="${entries_tests}- $display
+" ;;
+    ci)       entries_ci="${entries_ci}- $display
+" ;;
+    build)    entries_build="${entries_build}- $display
+" ;;
+    *)        entries_chores="${entries_chores}- $display
+" ;;
+  esac
+done <<< "$COMMITS"
+
+# Assemble the changelog block
+TODAY=$(date +%Y-%m-%d)
+CHANGELOG_BLOCK="## [v$NEW_VERSION] — $TODAY
+"
+
+for key in breaking features fixes perf refactor docs style tests ci build chores; do
+  eval "content=\"\$entries_$key\""
+  if [ -n "$content" ]; then
+    CHANGELOG_BLOCK="${CHANGELOG_BLOCK}
+### ${SECTIONS[$key]}
+
+$content"
+  fi
+done
+
+# ─── Step 6 — Dry run check ───────────────────────────────────────────────────
+if [ "$DRY_RUN" = true ]; then
+  echo ""
+  echo "═══════════════════════════════════════════════════════════════"
+  echo "  DRY RUN — no changes made"
+  echo "═══════════════════════════════════════════════════════════════"
+  echo "  Version:  $CURRENT_VERSION → $NEW_VERSION"
+  echo "  Bump:     $BUMP_TYPE"
+  echo "  Commits:  $COMMIT_COUNT since ${LAST_TAG:-<none>}"
+  echo "  Branch:   $BRANCH"
+  echo ""
+  echo "  Files that would change:"
+  echo "    - package.json  (version → $NEW_VERSION)"
+  echo "    - CHANGELOG.md  (new entry below)"
+  echo ""
+  echo "  Tag that would be created: v$NEW_VERSION"
+  echo ""
+  echo "  Changelog entry:"
+  echo "───────────────────────────────────────────────────────────────"
+  echo "$CHANGELOG_BLOCK"
+  echo "───────────────────────────────────────────────────────────────"
+  exit 0
+fi
+
+# ─── Step 7 — Bump version in package.json ────────────────────────────────────
+npm pkg set version="$NEW_VERSION"
+
+node -e "
+const v = require('./package.json').version;
+if (v !== '$NEW_VERSION') {
+  console.error('✗ version mismatch: expected $NEW_VERSION, got ' + v);
+  process.exit(1);
+}
+console.log('✓ version bumped to $NEW_VERSION');
+"
+
+# ─── Step 8 — Write CHANGELOG.md ──────────────────────────────────────────────
+if [ -f CHANGELOG.md ]; then
+  # Prepend after the first heading line
+  {
+    head -n 1 CHANGELOG.md
+    echo ""
+    echo "$CHANGELOG_BLOCK"
+    tail -n +2 CHANGELOG.md
+  } > CHANGELOG.md.tmp && mv CHANGELOG.md.tmp CHANGELOG.md
+else
+  {
+    echo "# Changelog"
+    echo ""
+    echo "All notable changes to this project are documented in this file."
+    echo ""
+    echo "$CHANGELOG_BLOCK"
+  } > CHANGELOG.md
+fi
+
+ok "CHANGELOG.md updated"
+
+# ─── Step 9 — Read co-author config ───────────────────────────────────────────
+CO_AUTHOR="pi-mono <261679550+pi-mono@users.noreply.github.com>"
+if [ -f .pi/auto-commit.json ]; then
+  CO_AUTHOR=$(node -e "
+    const fs = require('fs');
+    const cfg = JSON.parse(fs.readFileSync('.pi/auto-commit.json', 'utf8'));
+    const ca = cfg.coAuthor || {};
+    console.log((ca.login || 'pi-mono') + ' <' + (ca.email || '261679550+pi-mono@users.noreply.github.com') + '>');
+  " 2>/dev/null) || true
+fi
+
+# ─── Step 10 — Commit ─────────────────────────────────────────────────────────
+git add package.json CHANGELOG.md
+
+COMMIT_BODY=$(cat <<EOF
+- Bump version in package.json
+- Add changelog entry for v$NEW_VERSION
+
+Commits included:
+$(echo "$COMMITS" | sed 's/^/- /')
+EOF
+)
+
+git commit -m "chore(release): bump to v$NEW_VERSION" \
+           -m "$COMMIT_BODY" \
+           -m "Co-authored-by: $CO_AUTHOR"
+
+ok "Committed version bump + changelog"
+
+# ─── Step 11 — Create and push tag ────────────────────────────────────────────
+TAG_BODY=$(cat <<EOF
+Release v$NEW_VERSION — $BUMP_TYPE bump
+
+$COMMITS
+EOF
+)
+
+git tag -a "v$NEW_VERSION" -m "$TAG_BODY"
+ok "Created tag v$NEW_VERSION"
+
+git push origin "v$NEW_VERSION"
+ok "Pushed tag v$NEW_VERSION to origin"
+
+# ─── Step 12 — Optionally push branch commit ──────────────────────────────────
+echo ""
+read -rp "Push the version-bump commit to the current branch ($BRANCH) too? [Y/n] " PUSH_BRANCH
+if [[ "$PUSH_BRANCH" =~ ^[Yy]?$ ]]; then
+  git push origin "$BRANCH"
+  ok "Pushed commit to $BRANCH"
+else
+  echo "Skipped branch push."
+fi
+
+# ─── Step 13 — Report ─────────────────────────────────────────────────────────
+echo ""
+echo "═══════════════════════════════════════════════════════════════"
+echo "  ✓ Released v$NEW_VERSION ($BUMP_TYPE)"
+echo "═══════════════════════════════════════════════════════════════"
+echo "  Tag:      v$NEW_VERSION — pushed to origin"
+echo "  Commit:   $(git rev-parse --short HEAD)"
+echo "  Branch:   $BRANCH"
+echo ""
+echo "  Workflows triggered:"
+echo "    - .github/workflows/publish-github-packages.yml"
+echo "    - .github/workflows/publish-npm.yml"
+echo ""
+echo "  Changelog: CHANGELOG.md updated"
+echo "  Monitor:   https://github.com/aryaniyaps/ultimate-pi/actions"
+echo "═══════════════════════════════════════════════════════════════"

package/.pi/scripts/sentrux-rules-sync.mjs

@@ -13,16 +13,26 @@ import { fileURLToPath } from "node:url";
 import { createHash } from "node:crypto";
 import { spawn } from "node:child_process";
 
-const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+const UP_PKG = join(dirname(fileURLToPath(import.meta.url)), "..", "..");
+/** Target project root (consumer repo). Default: process.cwd(). */
+const PROJECT_ROOT =
+	process.argv.find((a, i) => i >= 2 && !a.startsWith("-")) || process.cwd();
 const MANIFEST = join(
-	ROOT,
+	PROJECT_ROOT,
 	".pi",
 	"harness",
 	"sentrux",
 	"architecture.manifest.json",
 );
-const RULES_PATH = join(ROOT, ".sentrux", "rules.toml");
-const META_PATH = join(ROOT, ".sentrux", ".harness-rules-meta.json");
+const MANIFEST_TEMPLATE = join(
+	UP_PKG,
+	".pi",
+	"harness",
+	"sentrux",
+	"architecture.manifest.json",
+);
+const RULES_PATH = join(PROJECT_ROOT, ".sentrux", "rules.toml");
+const META_PATH = join(PROJECT_ROOT, ".sentrux", ".harness-rules-meta.json");
 
 const MANAGED_START = "# --- harness:managed:start ---";
 const MANAGED_END = "# --- harness:managed:end ---";
@@ -113,7 +123,7 @@ async function fileExists(path) {
 async function runSentruxCheck() {
 	return new Promise((resolve) => {
 		const child = spawn("sentrux", ["check", "."], {
-			cwd: ROOT,
+			cwd: PROJECT_ROOT,
 			stdio: ["ignore", "pipe", "pipe"],
 		});
 		let out = "";
@@ -136,7 +146,19 @@ async function main() {
 	const strict = process.argv.includes("--strict");
 
 	if (!(await fileExists(MANIFEST))) {
-		fail(`missing manifest ${MANIFEST}`);
+		if (await fileExists(MANIFEST_TEMPLATE)) {
+			await mkdir(dirname(MANIFEST), { recursive: true });
+			await writeFile(
+				MANIFEST,
+				await readFile(MANIFEST_TEMPLATE, "utf-8"),
+				"utf-8",
+			);
+			console.log(
+				`sentrux-rules-sync: seeded manifest from package -> ${MANIFEST}`,
+			);
+		} else {
+			fail(`missing manifest ${MANIFEST} (and no template in package)`);
+		}
 	}
 
 	const manifestRaw = await readFile(MANIFEST, "utf-8");
@@ -171,7 +193,7 @@ async function main() {
 			"rules.toml out of date — run node \"$UP_PKG/.pi/scripts/sentrux-rules-sync.mjs\" --force (see .pi/scripts/README.md for UP_PKG)",
 		);
 	} else {
-		await mkdir(join(ROOT, ".sentrux"), { recursive: true });
+		await mkdir(join(PROJECT_ROOT, ".sentrux"), { recursive: true });
 		const next = mergeRules(existing, managedBlock);
 		await writeFile(RULES_PATH, next, "utf-8");
 		meta = {

package/.pi/settings.example.json

@@ -7,7 +7,6 @@
 		"npm:ultimate-pi",
 		"npm:@posthog/pi",
 		"npm:context-mode",
-		"npm:@tintinweb/pi-subagents",
 		"npm:@sting8k/pi-vcc"
 	]
 }

package/.sentrux/rules.toml

@@ -35,7 +35,7 @@ order = 2
 
 [[layers]]
 name = "agents"
-paths = [".pi/agents/*", ".pi/prompts/*", ".agents/skills/*"]
+paths = [".pi/agents/*", ".pi/harness/agents.manifest.json", ".pi/prompts/*", ".agents/skills/*"]
 order = 3
 # Agent definitions, prompts, and skills
 

package/AGENTS.md

@@ -30,7 +30,7 @@ Created: 2026-05-14
 - Harness context: **context-mode only** — never lean-ctx on harness paths (see harness-context skill)
 - `graphify update .` after significant code changes
 - ast-grep (`sg`) is the default code search tool — use `sg -p 'pattern'` for structural search, never grep for code
-- Self-hosted Firecrawl at http://localhost:3002 (FIRECRAWL_API_URL)
+- Web fetch/search via `python3 "$UP_PKG/.pi/scripts/harness-web.py"` (Scrapling; see scrapling-web skill)
 
 ## graphify
 

package/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 All notable changes to this project are documented in this file.
 
+## [v0.4.0] — 2026-05-16
+
+### ⚠️ Breaking Changes
+
+- **Firecrawl removed**: deleted self-hosted `firecrawl/` Docker stack and all `firecrawl*` agent skills. Web fetch/search now uses **`harness-web`** (Scrapling). Artifact dir is **`.web/`** (not `.firecrawl/`). Remove `FIRECRAWL_API_KEY` / `FIRECRAWL_API_URL`; use `uv tool install "scrapling[fetchers]"` and `scrapling install` per `/harness-setup`.
+- **Harness web CLI**: `python3 "$UP_PKG/.pi/scripts/harness-web.py" search|scrape|map|bulk-scrape` — see `.agents/skills/scrapling-web/SKILL.md`.
+
+### ✨ Features
+
+- **harness-web**: Scrapling-backed search (DuckDuckGo HTML SERP) and scrape (`StealthyFetcher` default; `--fast` for static HTTP). Modules under `.pi/scripts/harness_web/`.
+- **Harness subagents**: vendored harness-subagents extension and Sentrux bootstrap (`#125`).
+
 ## [v0.3.1] — 2026-05-15
 
 ### 🐛 Fixes