ultimate-pi 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/.agents/skills/harness-decisions/SKILL.md +15 -0
  2. package/.agents/skills/scrapling-web/SKILL.md +45 -40
  3. package/.agents/skills/wiki-autoresearch/SKILL.md +3 -3
  4. package/.pi/PACKAGING.md +3 -2
  5. package/.pi/SYSTEM.md +12 -13
  6. package/.pi/agents/pi-pi/agent-expert.md +3 -3
  7. package/.pi/extensions/harness-web-guard.ts +95 -0
  8. package/.pi/extensions/harness-web-tools.ts +209 -0
  9. package/.pi/extensions/lib/harness-vcc-settings.ts +50 -0
  10. package/.pi/extensions/lib/harness-web/run-cli.ts +92 -0
  11. package/.pi/extensions/ultimate-pi-vcc.ts +17 -0
  12. package/.pi/harness/docs/adrs/0030-inhouse-vcc-compaction.md +40 -0
  13. package/.pi/harness/docs/adrs/README.md +1 -0
  14. package/.pi/harness/env.harness.template +3 -1
  15. package/.pi/prompts/harness-setup.md +48 -2
  16. package/.pi/scripts/harness-cli-verify.sh +12 -3
  17. package/.pi/scripts/harness-searxng-bootstrap.mjs +270 -0
  18. package/.pi/scripts/harness-web-search.md +24 -5
  19. package/.pi/scripts/harness-web.py +24 -7
  20. package/.pi/scripts/harness_web/config.py +37 -3
  21. package/.pi/scripts/harness_web/output.py +8 -2
  22. package/.pi/scripts/harness_web/search.py +22 -0
  23. package/.pi/scripts/harness_web/search_ddg.py +1 -5
  24. package/.pi/scripts/harness_web/search_searxng.py +100 -0
  25. package/.pi/scripts/vendor-pi-vcc-settings.stub.ts +8 -0
  26. package/.pi/scripts/vendor-sync-pi-vcc.sh +40 -0
  27. package/.pi/settings.example.json +1 -6
  28. package/CHANGELOG.md +20 -6
  29. package/THIRD_PARTY_NOTICES.md +8 -22
  30. package/package.json +7 -6
  31. package/vendor/pi-vcc/README.md +215 -0
  32. package/vendor/pi-vcc/UPSTREAM_PIN.md +12 -0
  33. package/vendor/pi-vcc/demo.gif +0 -0
  34. package/vendor/pi-vcc/index.ts +12 -0
  35. package/vendor/pi-vcc/package.json +26 -0
  36. package/vendor/pi-vcc/scripts/audit-sessions.ts +88 -0
  37. package/vendor/pi-vcc/scripts/benchmark-real-sessions.ts +25 -0
  38. package/vendor/pi-vcc/scripts/compare-before-after.ts +36 -0
  39. package/vendor/pi-vcc/scripts/dump-branch-output.ts +20 -0
  40. package/vendor/pi-vcc/src/commands/pi-vcc.ts +36 -0
  41. package/vendor/pi-vcc/src/commands/vcc-recall.ts +65 -0
  42. package/vendor/pi-vcc/src/core/brief.ts +381 -0
  43. package/vendor/pi-vcc/src/core/build-sections.ts +79 -0
  44. package/vendor/pi-vcc/src/core/content.ts +60 -0
  45. package/vendor/pi-vcc/src/core/filter-noise.ts +42 -0
  46. package/vendor/pi-vcc/src/core/format-recall.ts +27 -0
  47. package/vendor/pi-vcc/src/core/format.ts +49 -0
  48. package/vendor/pi-vcc/src/core/lineage.ts +26 -0
  49. package/vendor/pi-vcc/src/core/load-messages.ts +41 -0
  50. package/vendor/pi-vcc/src/core/normalize.ts +66 -0
  51. package/vendor/pi-vcc/src/core/recall-scope.ts +14 -0
  52. package/vendor/pi-vcc/src/core/render-entries.ts +55 -0
  53. package/vendor/pi-vcc/src/core/report.ts +237 -0
  54. package/vendor/pi-vcc/src/core/sanitize.ts +5 -0
  55. package/vendor/pi-vcc/src/core/search-entries.ts +221 -0
  56. package/vendor/pi-vcc/src/core/settings.ts +8 -0
  57. package/vendor/pi-vcc/src/core/skill-collapse.ts +35 -0
  58. package/vendor/pi-vcc/src/core/summarize.ts +157 -0
  59. package/vendor/pi-vcc/src/core/tool-args.ts +14 -0
  60. package/vendor/pi-vcc/src/details.ts +7 -0
  61. package/vendor/pi-vcc/src/extract/commits.ts +69 -0
  62. package/vendor/pi-vcc/src/extract/files.ts +80 -0
  63. package/vendor/pi-vcc/src/extract/goals.ts +79 -0
  64. package/vendor/pi-vcc/src/extract/preferences.ts +55 -0
  65. package/vendor/pi-vcc/src/hooks/before-compact.ts +314 -0
  66. package/vendor/pi-vcc/src/sections.ts +12 -0
  67. package/vendor/pi-vcc/src/tools/recall.ts +109 -0
  68. package/vendor/pi-vcc/src/types.ts +14 -0
  69. package/vendor/pi-vcc/tests/before-compact-hook.test.ts +204 -0
  70. package/vendor/pi-vcc/tests/before-compact.test.ts +145 -0
  71. package/vendor/pi-vcc/tests/brief.test.ts +206 -0
  72. package/vendor/pi-vcc/tests/build-sections.test.ts +59 -0
  73. package/vendor/pi-vcc/tests/compile.test.ts +80 -0
  74. package/vendor/pi-vcc/tests/content.test.ts +31 -0
  75. package/vendor/pi-vcc/tests/extract-goals.test.ts +86 -0
  76. package/vendor/pi-vcc/tests/extract-preferences.test.ts +30 -0
  77. package/vendor/pi-vcc/tests/filter-noise.test.ts +61 -0
  78. package/vendor/pi-vcc/tests/fixtures.ts +61 -0
  79. package/vendor/pi-vcc/tests/format-recall.test.ts +30 -0
  80. package/vendor/pi-vcc/tests/format.test.ts +62 -0
  81. package/vendor/pi-vcc/tests/lineage.test.ts +33 -0
  82. package/vendor/pi-vcc/tests/load-messages.test.ts +51 -0
  83. package/vendor/pi-vcc/tests/normalize.test.ts +97 -0
  84. package/vendor/pi-vcc/tests/real-sessions.test.ts +38 -0
  85. package/vendor/pi-vcc/tests/recall-expand.test.ts +15 -0
  86. package/vendor/pi-vcc/tests/recall-scope.test.ts +32 -0
  87. package/vendor/pi-vcc/tests/recall-tool-scope.test.ts +67 -0
  88. package/vendor/pi-vcc/tests/render-entries.test.ts +62 -0
  89. package/vendor/pi-vcc/tests/report.test.ts +44 -0
  90. package/vendor/pi-vcc/tests/sanitize.test.ts +24 -0
  91. package/vendor/pi-vcc/tests/search-entries.test.ts +144 -0
  92. package/vendor/pi-vcc/tests/support/load-session.ts +23 -0
  93. package/vendor/pi-vcc/tests/support/real-sessions.ts +51 -0
  94. package/.pi/pi-vcc-config.json +0 -4
@@ -36,7 +36,7 @@ if str(SCRIPT_DIR) not in sys.path:
36
36
  from harness_web.config import HarnessWebConfig, load_config # noqa: E402
37
37
  from harness_web.output import write_search_results # noqa: E402
38
38
  from harness_web.scrape import bulk_scrape, map_url, scrape_url # noqa: E402
39
- from harness_web.search_ddg import search_ddg # noqa: E402
39
+ from harness_web.search import search # noqa: E402
40
40
 
41
41
  DEFAULT_WEB_DIR = ".web"
42
42
 
@@ -47,8 +47,8 @@ def _default_out(sub: str) -> Path:
47
47
 
48
48
  def cmd_search(args: argparse.Namespace, config: HarnessWebConfig) -> int:
49
49
  out = Path(args.output or _default_out("search.json"))
50
- results = search_ddg(args.query, limit=args.limit, config=config)
51
- write_search_results(out, results, args.query)
50
+ results = search(args.query, limit=args.limit, config=config)
51
+ write_search_results(out, results, args.query, engine=config.search_engine)
52
52
  print(f"wrote {out} ({len(results)} results)")
53
53
  return 0
54
54
 
@@ -76,6 +76,20 @@ def cmd_map(args: argparse.Namespace, config: HarnessWebConfig) -> int:
76
76
  return 0
77
77
 
78
78
 
79
+ def cmd_status(_args: argparse.Namespace, config: HarnessWebConfig) -> int:
80
+ import json
81
+
82
+ payload = {
83
+ "search_engine": config.search_engine,
84
+ "searxng_url": config.searxng_url,
85
+ "fetch_mode": config.fetch_mode,
86
+ "script": str(Path(__file__).resolve()),
87
+ "bootstrap": "ok",
88
+ }
89
+ print(json.dumps(payload, indent=2))
90
+ return 0
91
+
92
+
79
93
  def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
80
94
  sleep_sec = args.sleep if args.sleep is not None else config.rate_limit_ms / 1000.0
81
95
  if args.urls:
@@ -86,8 +100,8 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
86
100
  data = json.loads(Path(args.from_search).read_text(encoding="utf-8"))
87
101
  urls = [item["url"] for item in data.get("data", {}).get("web", []) if item.get("url")]
88
102
  else:
89
- urls = search_ddg(args.query, limit=args.limit, config=config)
90
- urls = [r["url"] for r in urls]
103
+ serp = search(args.query, limit=args.limit, config=config)
104
+ urls = [r["url"] for r in serp]
91
105
 
92
106
  if not urls:
93
107
  print("bulk-scrape: no URLs to fetch", file=sys.stderr)
@@ -111,11 +125,11 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
111
125
  def build_parser() -> argparse.ArgumentParser:
112
126
  p = argparse.ArgumentParser(
113
127
  prog="harness-web",
114
- description="Harness web layer: search (DDG HTML) and scrape (Scrapling).",
128
+ description="Harness web layer: search (DDG HTML or SearXNG) and scrape (Scrapling).",
115
129
  )
116
130
  sub = p.add_subparsers(dest="command", required=True)
117
131
 
118
- ps = sub.add_parser("search", help="Search via DuckDuckGo HTML SERP")
132
+ ps = sub.add_parser("search", help="Search via configured SERP (HARNESS_WEB_SEARCH_ENGINE)")
119
133
  ps.add_argument("query", help="Search query")
120
134
  ps.add_argument("-o", "--output", help="JSON output path (default: .web/search.json)")
121
135
  ps.add_argument("--limit", type=int, default=5)
@@ -160,6 +174,9 @@ def build_parser() -> argparse.ArgumentParser:
160
174
  pm.add_argument("--fast", action="store_true")
161
175
  pm.set_defaults(func=cmd_map)
162
176
 
177
+ pst = sub.add_parser("status", help="Print harness-web config as JSON (setup/diagnostics)")
178
+ pst.set_defaults(func=cmd_status)
179
+
163
180
  return p
164
181
 
165
182
 
@@ -6,6 +6,8 @@ import os
6
6
  from dataclasses import dataclass
7
7
  from urllib.parse import urlparse
8
8
 
9
+ SUPPORTED_SEARCH_ENGINES = frozenset({"ddg_html", "searxng"})
10
+
9
11
 
10
12
  def _int_env(name: str, default: int) -> int:
11
13
  raw = os.environ.get(name, "").strip()
@@ -24,6 +26,18 @@ def _fetch_mode() -> str:
24
26
  return "stealth"
25
27
 
26
28
 
29
+ def _normalize_searxng_url(raw: str) -> str:
30
+ url = raw.strip().rstrip("/")
31
+ if not url:
32
+ return ""
33
+ parsed = urlparse(url)
34
+ if parsed.scheme not in ("http", "https") or not parsed.netloc:
35
+ raise SystemExit(
36
+ f"Invalid HARNESS_WEB_SEARXNG_URL={raw!r} — expected http(s)://host[:port]"
37
+ )
38
+ return url
39
+
40
+
27
41
  _STATIC_HOSTS = frozenset(
28
42
  {
29
43
  "example.com",
@@ -50,6 +64,7 @@ def host_is_static(url: str) -> bool:
50
64
  class HarnessWebConfig:
51
65
  fetch_mode: str
52
66
  search_engine: str
67
+ searxng_url: str | None
53
68
  proxy: str | None
54
69
  rate_limit_ms: int
55
70
  timeout_ms: int
@@ -68,13 +83,32 @@ class HarnessWebConfig:
68
83
  return False
69
84
 
70
85
 
86
+ def validate_search_config(config: HarnessWebConfig) -> None:
87
+ engine = config.search_engine
88
+ if engine not in SUPPORTED_SEARCH_ENGINES:
89
+ supported = ", ".join(sorted(SUPPORTED_SEARCH_ENGINES))
90
+ raise SystemExit(
91
+ f"Unsupported HARNESS_WEB_SEARCH_ENGINE={engine!r} (supported: {supported})"
92
+ )
93
+ if engine == "searxng" and not config.searxng_url:
94
+ raise SystemExit(
95
+ "HARNESS_WEB_SEARCH_ENGINE=searxng requires HARNESS_WEB_SEARXNG_URL "
96
+ "(e.g. http://127.0.0.1:8080). Run /harness-setup and choose SearXNG, or set both in .env."
97
+ )
98
+
99
+
71
100
  def load_config() -> HarnessWebConfig:
72
101
  proxy = os.environ.get("HARNESS_WEB_PROXY", "").strip() or None
73
- return HarnessWebConfig(
102
+ engine = os.environ.get("HARNESS_WEB_SEARCH_ENGINE", "ddg_html").strip() or "ddg_html"
103
+ searx_raw = os.environ.get("HARNESS_WEB_SEARXNG_URL", "").strip()
104
+ searxng_url = _normalize_searxng_url(searx_raw) if searx_raw else None
105
+ config = HarnessWebConfig(
74
106
  fetch_mode=_fetch_mode(),
75
- search_engine=os.environ.get("HARNESS_WEB_SEARCH_ENGINE", "ddg_html").strip()
76
- or "ddg_html",
107
+ search_engine=engine,
108
+ searxng_url=searxng_url,
77
109
  proxy=proxy,
78
110
  rate_limit_ms=_int_env("HARNESS_WEB_RATE_LIMIT_MS", 2000),
79
111
  timeout_ms=_int_env("HARNESS_WEB_TIMEOUT_MS", 30000),
80
112
  )
113
+ validate_search_config(config)
114
+ return config
@@ -18,13 +18,19 @@ def write_json(path: Path, payload: Any) -> None:
18
18
  path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
19
19
 
20
20
 
21
- def write_search_results(path: Path, results: list[dict[str, str]], query: str) -> None:
21
+ def write_search_results(
22
+ path: Path,
23
+ results: list[dict[str, str]],
24
+ query: str,
25
+ *,
26
+ engine: str,
27
+ ) -> None:
22
28
  """Firecrawl-compatible envelope: data.web[].url|title|description."""
23
29
  write_json(
24
30
  path,
25
31
  {
26
32
  "query": query,
27
- "engine": "ddg_html",
33
+ "engine": engine,
28
34
  "data": {
29
35
  "web": [
30
36
  {
@@ -0,0 +1,22 @@
1
+ """Route harness-web search to the configured SERP backend."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .config import HarnessWebConfig, validate_search_config
6
+ from .search_ddg import search_ddg
7
+ from .search_searxng import search_searxng
8
+
9
+
10
+ def search(
11
+ query: str,
12
+ *,
13
+ limit: int,
14
+ config: HarnessWebConfig,
15
+ ) -> list[dict[str, str]]:
16
+ validate_search_config(config)
17
+ engine = config.search_engine
18
+ if engine == "searxng":
19
+ return search_searxng(query, limit=limit, config=config)
20
+ if engine == "ddg_html":
21
+ return search_ddg(query, limit=limit, config=config)
22
+ raise SystemExit(f"Unsupported HARNESS_WEB_SEARCH_ENGINE={engine!r}")
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ from typing import Any
5
6
  from urllib.parse import parse_qs, unquote, urlparse
6
7
 
7
8
  from scrapling.fetchers import Fetcher, StealthyFetcher
@@ -63,11 +64,6 @@ def search_ddg(
63
64
  config: HarnessWebConfig,
64
65
  impersonate: bool = True,
65
66
  ) -> list[dict[str, str]]:
66
- if config.search_engine != "ddg_html":
67
- raise SystemExit(
68
- f"Unsupported HARNESS_WEB_SEARCH_ENGINE={config.search_engine!r} (only ddg_html)"
69
- )
70
-
71
67
  kwargs: dict = {
72
68
  "params": {"q": query},
73
69
  "timeout": config.timeout_sec,
@@ -0,0 +1,100 @@
1
+ """SearXNG JSON search API (self-hosted instances)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import ssl
7
+ from typing import Any
8
+ from urllib.error import HTTPError, URLError
9
+ from urllib.parse import urlencode
10
+ from urllib.request import ProxyHandler, Request, build_opener, urlopen
11
+
12
+ from .config import HarnessWebConfig
13
+
14
+
15
+ def _open_url(url: str, *, config: HarnessWebConfig) -> tuple[int, str]:
16
+ req = Request(
17
+ url,
18
+ headers={"Accept": "application/json", "User-Agent": "ultimate-pi-harness-web/1.0"},
19
+ method="GET",
20
+ )
21
+ if config.proxy:
22
+ opener = build_opener(ProxyHandler({"http": config.proxy, "https": config.proxy}))
23
+ resp = opener.open(req, timeout=config.timeout_sec)
24
+ else:
25
+ ctx = ssl.create_default_context()
26
+ resp = urlopen(req, timeout=config.timeout_sec, context=ctx)
27
+ try:
28
+ status = getattr(resp, "status", 200) or 200
29
+ body = resp.read().decode("utf-8", errors="replace")
30
+ return status, body
31
+ finally:
32
+ resp.close()
33
+
34
+
35
+ def _parse_results(payload: Any, limit: int) -> list[dict[str, str]]:
36
+ raw = payload.get("results") if isinstance(payload, dict) else None
37
+ if not isinstance(raw, list):
38
+ return []
39
+ out: list[dict[str, str]] = []
40
+ for item in raw:
41
+ if len(out) >= limit:
42
+ break
43
+ if not isinstance(item, dict):
44
+ continue
45
+ url = (item.get("url") or "").strip()
46
+ if not url.startswith("http"):
47
+ continue
48
+ title = (item.get("title") or "").strip()
49
+ description = (item.get("content") or item.get("snippet") or "").strip()
50
+ out.append({"url": url, "title": title, "description": description})
51
+ return out
52
+
53
+
54
+ def search_searxng(
55
+ query: str,
56
+ *,
57
+ limit: int,
58
+ config: HarnessWebConfig,
59
+ ) -> list[dict[str, str]]:
60
+ base = config.searxng_url
61
+ if not base:
62
+ raise SystemExit("HARNESS_WEB_SEARXNG_URL is not set")
63
+
64
+ qs = urlencode({"q": query, "format": "json", "pageno": "1"})
65
+ url = f"{base}/search?{qs}"
66
+
67
+ try:
68
+ status, body = _open_url(url, config=config)
69
+ except HTTPError as err:
70
+ status = err.code
71
+ body = err.read().decode("utf-8", errors="replace") if err.fp else ""
72
+ except URLError as err:
73
+ raise SystemExit(
74
+ f"SearXNG request failed ({err.reason}). "
75
+ f"Is the instance running at {base}? "
76
+ "Run: node \"$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs\""
77
+ ) from err
78
+
79
+ if status == 403:
80
+ raise SystemExit(
81
+ "SearXNG returned 403 for format=json. Enable json under search.formats "
82
+ "in settings.yml (see .searxng/core-config/settings.yml or SearXNG docs)."
83
+ )
84
+ if status != 200:
85
+ snippet = body[:200].replace("\n", " ")
86
+ raise SystemExit(f"SearXNG search failed (HTTP {status}): {snippet}")
87
+
88
+ try:
89
+ payload = json.loads(body)
90
+ except json.JSONDecodeError as err:
91
+ raise SystemExit(f"SearXNG returned non-JSON response from {url}") from err
92
+
93
+ results = _parse_results(payload, limit)
94
+ if not results and isinstance(payload, dict):
95
+ unresponsive = payload.get("unresponsive_engines")
96
+ if unresponsive:
97
+ raise SystemExit(
98
+ f"SearXNG returned no results; upstream engines unresponsive: {unresponsive}"
99
+ )
100
+ return results
@@ -0,0 +1,8 @@
1
+ /**
2
+ * ultimate-pi harness settings (env-only). Re-exported for vendored pi-vcc layout.
3
+ */
4
+ export type { PiVccSettings } from "../../../../.pi/extensions/lib/harness-vcc-settings.js";
5
+ export {
6
+ loadSettings,
7
+ scaffoldSettings,
8
+ } from "../../../../.pi/extensions/lib/harness-vcc-settings.js";
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/env bash
2
+ # Re-fetch upstream pi-vcc and re-apply ultimate-pi patches.
3
+ set -euo pipefail
4
+ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
5
+ VEND="$ROOT/vendor/pi-vcc"
6
+ SETTINGS_STUB="$ROOT/.pi/scripts/vendor-pi-vcc-settings.stub.ts"
7
+
8
+ rm -rf "$VEND"
9
+ git clone --depth 1 https://github.com/sting8k/pi-vcc.git "$VEND"
10
+ COMMIT="$(git -C "$VEND" rev-parse HEAD)"
11
+ rm -rf "$VEND/.git"
12
+
13
+ # Env-only settings (no JSON config files)
14
+ cp "$SETTINGS_STUB" "$VEND/src/core/settings.ts"
15
+
16
+ # Drop config scaffolding on extension load
17
+ sed -i '/scaffoldSettings/d' "$VEND/index.ts"
18
+ sed -i '/import.*scaffoldSettings/d' "$VEND/index.ts"
19
+
20
+ # Telemetry / compaction details label
21
+ sed -i 's/compactor: "pi-vcc";/compactor: "pi-vcc" | "ultimate-pi-vcc";/' "$VEND/src/details.ts"
22
+ sed -i 's/compactor: "pi-vcc"/compactor: "ultimate-pi-vcc"/' "$VEND/src/hooks/before-compact.ts"
23
+
24
+ cat >"$VEND/UPSTREAM_PIN.md" <<EOF
25
+ # Vendored \`pi-vcc\`
26
+
27
+ - **Repository:** https://github.com/sting8k/pi-vcc
28
+ - **Conceptual basis:** [lllyasviel/VCC](https://github.com/lllyasviel/VCC) (View-oriented Conversation Compiler)
29
+ - **License:** MIT (see upstream repository)
30
+ - **Pinned upstream commit:** \`$COMMIT\`
31
+ - **Local changes:**
32
+ - \`src/core/settings.ts\` re-exports env-only [\`harness-vcc-settings\`](../../.pi/extensions/lib/harness-vcc-settings.ts) (\`HARNESS_VCC_COMPACTION\`, \`HARNESS_VCC_DEBUG\`)
33
+ - No \`scaffoldSettings\` / no \`PI_VCC_CONFIG_PATH\`
34
+ - Compaction \`details.compactor\` is \`ultimate-pi-vcc\`
35
+
36
+ **Refresh upstream:** run \`npm run vendor:sync-vcc\` from ultimate-pi root.
37
+ EOF
38
+
39
+ rm -f "$VEND/package-lock.json"
40
+ echo "✓ Vendor refreshed at $VEND (commit $COMMIT)"
@@ -3,10 +3,5 @@
3
3
  "enabled": true,
4
4
  "maxRetries": 3
5
5
  },
6
- "packages": [
7
- "npm:ultimate-pi",
8
- "npm:@posthog/pi",
9
- "npm:context-mode",
10
- "npm:@sting8k/pi-vcc"
11
- ]
6
+ "packages": ["npm:ultimate-pi", "npm:@posthog/pi", "npm:context-mode"]
12
7
  }
package/CHANGELOG.md CHANGED
@@ -2,17 +2,31 @@
2
2
 
3
3
  All notable changes to this project are documented in this file.
4
4
 
5
- ## [v0.4.0] — 2026-05-16
5
+ ## [Unreleased]
6
6
 
7
- ### ⚠️ Breaking Changes
7
+ ## [v0.5.0] 2026-05-17
8
8
 
9
- - **Firecrawl removed**: deleted self-hosted `firecrawl/` Docker stack and all `firecrawl*` agent skills. Web fetch/search now uses **`harness-web`** (Scrapling). Artifact dir is **`.web/`** (not `.firecrawl/`). Remove `FIRECRAWL_API_KEY` / `FIRECRAWL_API_URL`; use `uv tool install "scrapling[fetchers]"` and `scrapling install` per `/harness-setup`.
10
- - **Harness web CLI**: `python3 "$UP_PKG/.pi/scripts/harness-web.py" search|scrape|map|bulk-scrape` — see `.agents/skills/scrapling-web/SKILL.md`.
9
+ ### Features
10
+
11
+ - **web_search / web_fetch pi tools:** wrap `harness-web.py` with session injection and a bash guard so agents skip `UP_PKG` and scrapling import preflights.
12
+ - **SearXNG search backend:** pluggable `HARNESS_WEB_SEARCH_ENGINE` (`ddg_html` | `searxng`) with Docker bootstrap via `harness-searxng-bootstrap.mjs`.
13
+ - **harness-web status:** JSON config subcommand for setup and diagnostics.
14
+
15
+ ### 🔧 Chores
16
+
17
+ - Apply pre-commit format and refresh `graphify-out` after harness-web tools merge.
18
+
19
+ ## [v0.4.1] — 2026-05-17
11
20
 
12
21
  ### ✨ Features
13
22
 
14
- - **harness-web**: Scrapling-backed search (DuckDuckGo HTML SERP) and scrape (`StealthyFetcher` default; `--fast` for static HTTP). Modules under `.pi/scripts/harness_web/`.
15
- - **Harness subagents**: vendored harness-subagents extension and Sentrux bootstrap (`#125`).
23
+ - **In-house VCC compaction:** vendored [pi-vcc](https://github.com/sting8k/pi-vcc) (inspired by [lllyasviel/VCC](https://github.com/lllyasviel/VCC)); removed `@sting8k/pi-vcc` npm dependency. Configuration is **env-only** (`HARNESS_VCC_COMPACTION`, `HARNESS_VCC_DEBUG`) — no `PI_VCC_CONFIG_PATH` or JSON config files. VCC overrides `/compact` and auto-compaction by default; set `HARNESS_VCC_COMPACTION=false` for Pi LLM compaction. Refresh upstream: `npm run vendor:sync-vcc`.
24
+ - **Harness subagents:** vendored harness-subagents extension with Sentrux bootstrap agent and related harness wiring.
25
+ - **Harness web:** replace Firecrawl with Scrapling-based `harness-web` CLI for search and fetch.
26
+
27
+ ### 🔧 Chores
28
+
29
+ - Format `.pi/settings.json` / `.pi/settings.example.json` and refresh `graphify-out` after VCC merge.
16
30
 
17
31
  ## [v0.3.1] — 2026-05-15
18
32
 
@@ -1,30 +1,16 @@
1
1
  # Third-party notices
2
2
 
3
- ## Design references (not vendored)
4
-
5
- The in-house `ask_user` harness extension (`.pi/extensions/harness-ask-user.ts`) borrows **behavior and UX patterns only** from these projects. No code from them is bundled.
6
-
7
- | Project | License | Reference |
8
- |---------|---------|-----------|
9
- | [pi-ask-user](https://github.com/edlsh/pi-ask-user) | MIT | Overlay/inline modes, SelectList + Editor, custom renderCall/renderResult, headless fallback |
10
- | [@pi-unipi/ask-user](https://cdn.jsdelivr.net/npm/@pi-unipi/unipi@2.0.1/packages/ask-user/README.md) | (package docs) | Tool contract: `question`, `context`, `options`, `allowMultiple`, `allowFreeform` |
11
- | [rpiv-ask-user-question](https://github.com/juicesharp/rpiv-mono/tree/main/packages/rpiv-ask-user-question) | MIT | Rich option rows, decision-handshake policy text, structured answer envelope |
12
-
13
- ## @tintinweb/pi-subagents (vendored in harness-subagents)
14
-
15
- - **Project:** https://github.com/tintinweb/pi-subagents
16
- - **Version pinned:** 0.7.3 (source vendored under `.pi/extensions/lib/harness-subagents/vendored/`)
17
- - **License:** MIT
18
- - **Notes:** Adapted for `@mariozechner/pi-coding-agent` in ultimate-pi `harness-subagents` extension. Agent discovery replaced with package-root recursive loader.
19
-
20
- ## subagent-v2 reference (design only)
21
-
22
- - **Path:** `raw/references/subagents/extensions/subagent-v2/`
23
- - **License:** Treat as third-party reference; not shipped in npm `files`. Blackboard, supervisor, and isolated session patterns were ported into `harness-subagents` without copying the earendil-works Pi stack.
24
-
25
3
  ## pi-model-router (vendored)
26
4
 
27
5
  - **Project:** https://github.com/yeliu84/pi-model-router
28
6
  - **License:** MIT ([vendor/pi-model-router/LICENSE](vendor/pi-model-router/LICENSE))
29
7
  - **Pinned revision:** See [vendor/pi-model-router/UPSTREAM_PIN.md](vendor/pi-model-router/UPSTREAM_PIN.md)
30
8
  - ultimate-pi loads it from [`vendor/pi-model-router`](vendor/pi-model-router); import specifiers were adapted for `@mariozechner/pi-coding-agent` and related Pi packages.
9
+
10
+ ## pi-vcc (vendored)
11
+
12
+ - **Project:** https://github.com/sting8k/pi-vcc
13
+ - **Conceptual basis:** https://github.com/lllyasviel/VCC (View-oriented Conversation Compiler)
14
+ - **License:** MIT (see upstream repository)
15
+ - **Pinned revision:** See [vendor/pi-vcc/UPSTREAM_PIN.md](vendor/pi-vcc/UPSTREAM_PIN.md)
16
+ - ultimate-pi loads it from [`vendor/pi-vcc`](vendor/pi-vcc) via [`.pi/extensions/ultimate-pi-vcc.ts`](.pi/extensions/ultimate-pi-vcc.ts). Harness configuration is env-only: `HARNESS_VCC_COMPACTION`, `HARNESS_VCC_DEBUG` ([`.pi/extensions/lib/harness-vcc-settings.ts`](.pi/extensions/lib/harness-vcc-settings.ts)). Maintainer refresh: `npm run vendor:sync-vcc`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ultimate-pi",
3
- "version": "0.4.0",
3
+ "version": "0.5.0",
4
4
  "description": "Ultimate AI coding harness for pi.dev — extensible skills, Obsidian wiki knowledge layer, compressed context, deterministic output",
5
5
  "keywords": [
6
6
  "pi-package",
@@ -60,7 +60,6 @@
60
60
  ".pi/settings.example.json",
61
61
  ".pi/auto-commit.json",
62
62
  ".pi/mcp.json",
63
- ".pi/pi-vcc-config.json",
64
63
  ".pi/SYSTEM.md",
65
64
  ".pi/PACKAGING.md",
66
65
  "AGENTS.md",
@@ -68,21 +67,24 @@
68
67
  "CHANGELOG.md",
69
68
  "README.md",
70
69
  "THIRD_PARTY_NOTICES.md",
71
- "vendor/pi-model-router"
70
+ "vendor/pi-model-router",
71
+ "vendor/pi-vcc"
72
72
  ],
73
73
  "peerDependencies": {
74
74
  "@mariozechner/pi-coding-agent": "*"
75
75
  },
76
76
  "scripts": {
77
- "check:ts": "tsc --noEmit --target ES2023 --lib ES2023 --moduleResolution nodenext --module nodenext --skipLibCheck .pi/extensions/dotenv-loader.ts .pi/extensions/lib/posthog-node.d.ts .pi/extensions/lib/harness-posthog.ts .pi/extensions/lib/harness-paths.ts .pi/extensions/pi-model-router-harness.ts .pi/extensions/provider-payload-sanitize.ts .pi/extensions/harness-telemetry.ts .pi/extensions/harness-ask-user.ts .pi/extensions/lib/ask-user/schema.ts .pi/extensions/lib/ask-user/types.ts .pi/extensions/lib/ask-user/validate.ts .pi/extensions/lib/ask-user/dialog.ts .pi/extensions/lib/ask-user/fallback.ts .pi/extensions/lib/ask-user/render.ts .pi/extensions/trace-recorder.ts .pi/extensions/observation-bus.ts .pi/extensions/drift-monitor.ts .pi/extensions/sentrux-rules-sync.ts .pi/extensions/custom-header.ts .pi/extensions/lib/harness-subagents/agent-loader.ts .pi/extensions/lib/harness-subagents/agent-parser.ts .pi/extensions/lib/harness-subagents/agent-manifest.ts .pi/extensions/lib/harness-subagents/blackboard.ts .pi/extensions/lib/harness-subagents/blackboard-tool.ts .pi/extensions/lib/harness-subagents/spawn-policy.ts .pi/extensions/lib/harness-subagents/types-blackboard.ts",
77
+ "check:ts": "tsc --noEmit --target ES2023 --lib ES2023 --moduleResolution nodenext --module nodenext --skipLibCheck .pi/extensions/lib/harness-vcc-settings.ts .pi/extensions/dotenv-loader.ts .pi/extensions/lib/posthog-node.d.ts .pi/extensions/lib/harness-posthog.ts .pi/extensions/lib/harness-paths.ts .pi/extensions/pi-model-router-harness.ts .pi/extensions/provider-payload-sanitize.ts .pi/extensions/harness-telemetry.ts .pi/extensions/harness-ask-user.ts .pi/extensions/lib/ask-user/schema.ts .pi/extensions/lib/ask-user/types.ts .pi/extensions/lib/ask-user/validate.ts .pi/extensions/lib/ask-user/dialog.ts .pi/extensions/lib/ask-user/fallback.ts .pi/extensions/lib/ask-user/render.ts .pi/extensions/trace-recorder.ts .pi/extensions/observation-bus.ts .pi/extensions/drift-monitor.ts .pi/extensions/sentrux-rules-sync.ts .pi/extensions/custom-header.ts .pi/extensions/lib/harness-subagents/agent-loader.ts .pi/extensions/lib/harness-subagents/agent-parser.ts .pi/extensions/lib/harness-subagents/agent-manifest.ts .pi/extensions/lib/harness-subagents/blackboard.ts .pi/extensions/lib/harness-subagents/blackboard-tool.ts .pi/extensions/lib/harness-subagents/spawn-policy.ts .pi/extensions/lib/harness-subagents/types-blackboard.ts .pi/extensions/harness-web-tools.ts .pi/extensions/harness-web-guard.ts .pi/extensions/lib/harness-web/run-cli.ts",
78
78
  "vendor:sync-router": "bash .pi/scripts/vendor-sync-pi-model-router.sh",
79
+ "vendor:sync-vcc": "bash .pi/scripts/vendor-sync-pi-vcc.sh",
79
80
  "release": "bash .pi/scripts/release.sh",
80
81
  "lint": "biome check",
81
82
  "lint:fix": "biome check --fix",
82
83
  "format": "biome format --write",
83
84
  "format:check": "biome format",
84
85
  "prepare": "lefthook install",
85
- "test": "node --test test/harness-verify.test.mjs test/harness-ask-user.test.mjs test/harness-subagents-loader.test.mjs test/sentrux-rules-sync.test.mjs",
86
+ "test": "node --test test/harness-verify.test.mjs test/harness-ask-user.test.mjs test/harness-subagents-loader.test.mjs test/sentrux-rules-sync.test.mjs && npx -y tsx --test test/harness-vcc-settings.test.ts",
87
+ "test:vcc": "npx -y tsx --test vendor/pi-vcc/tests/*.test.ts",
86
88
  "harness:sentrux-bootstrap": "node .pi/scripts/harness-sentrux-bootstrap.mjs",
87
89
  "harness:sentrux-sync": "node .pi/scripts/sentrux-rules-sync.mjs --force",
88
90
  "test:integration": "npx -y tsx --test test/cursor-sdk-provider.integration.test.ts"
@@ -99,7 +101,6 @@
99
101
  },
100
102
  "dependencies": {
101
103
  "@posthog/pi": "latest",
102
- "@sting8k/pi-vcc": "^0.3.12",
103
104
  "asciify-image": "^0.1.10",
104
105
  "croner": "^9.0.0",
105
106
  "jimp": "^1.6.1",