ultimate-pi 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/harness-decisions/SKILL.md +15 -0
- package/.agents/skills/scrapling-web/SKILL.md +45 -40
- package/.agents/skills/wiki-autoresearch/SKILL.md +3 -3
- package/.pi/PACKAGING.md +3 -2
- package/.pi/SYSTEM.md +12 -13
- package/.pi/agents/pi-pi/agent-expert.md +3 -3
- package/.pi/extensions/harness-web-guard.ts +95 -0
- package/.pi/extensions/harness-web-tools.ts +209 -0
- package/.pi/extensions/lib/harness-vcc-settings.ts +50 -0
- package/.pi/extensions/lib/harness-web/run-cli.ts +92 -0
- package/.pi/extensions/ultimate-pi-vcc.ts +17 -0
- package/.pi/harness/docs/adrs/0030-inhouse-vcc-compaction.md +40 -0
- package/.pi/harness/docs/adrs/README.md +1 -0
- package/.pi/harness/env.harness.template +3 -1
- package/.pi/prompts/harness-setup.md +48 -2
- package/.pi/scripts/harness-cli-verify.sh +12 -3
- package/.pi/scripts/harness-searxng-bootstrap.mjs +270 -0
- package/.pi/scripts/harness-web-search.md +24 -5
- package/.pi/scripts/harness-web.py +24 -7
- package/.pi/scripts/harness_web/config.py +37 -3
- package/.pi/scripts/harness_web/output.py +8 -2
- package/.pi/scripts/harness_web/search.py +22 -0
- package/.pi/scripts/harness_web/search_ddg.py +1 -5
- package/.pi/scripts/harness_web/search_searxng.py +100 -0
- package/.pi/scripts/vendor-pi-vcc-settings.stub.ts +8 -0
- package/.pi/scripts/vendor-sync-pi-vcc.sh +40 -0
- package/.pi/settings.example.json +1 -6
- package/CHANGELOG.md +20 -6
- package/THIRD_PARTY_NOTICES.md +8 -22
- package/package.json +7 -6
- package/vendor/pi-vcc/README.md +215 -0
- package/vendor/pi-vcc/UPSTREAM_PIN.md +12 -0
- package/vendor/pi-vcc/demo.gif +0 -0
- package/vendor/pi-vcc/index.ts +12 -0
- package/vendor/pi-vcc/package.json +26 -0
- package/vendor/pi-vcc/scripts/audit-sessions.ts +88 -0
- package/vendor/pi-vcc/scripts/benchmark-real-sessions.ts +25 -0
- package/vendor/pi-vcc/scripts/compare-before-after.ts +36 -0
- package/vendor/pi-vcc/scripts/dump-branch-output.ts +20 -0
- package/vendor/pi-vcc/src/commands/pi-vcc.ts +36 -0
- package/vendor/pi-vcc/src/commands/vcc-recall.ts +65 -0
- package/vendor/pi-vcc/src/core/brief.ts +381 -0
- package/vendor/pi-vcc/src/core/build-sections.ts +79 -0
- package/vendor/pi-vcc/src/core/content.ts +60 -0
- package/vendor/pi-vcc/src/core/filter-noise.ts +42 -0
- package/vendor/pi-vcc/src/core/format-recall.ts +27 -0
- package/vendor/pi-vcc/src/core/format.ts +49 -0
- package/vendor/pi-vcc/src/core/lineage.ts +26 -0
- package/vendor/pi-vcc/src/core/load-messages.ts +41 -0
- package/vendor/pi-vcc/src/core/normalize.ts +66 -0
- package/vendor/pi-vcc/src/core/recall-scope.ts +14 -0
- package/vendor/pi-vcc/src/core/render-entries.ts +55 -0
- package/vendor/pi-vcc/src/core/report.ts +237 -0
- package/vendor/pi-vcc/src/core/sanitize.ts +5 -0
- package/vendor/pi-vcc/src/core/search-entries.ts +221 -0
- package/vendor/pi-vcc/src/core/settings.ts +8 -0
- package/vendor/pi-vcc/src/core/skill-collapse.ts +35 -0
- package/vendor/pi-vcc/src/core/summarize.ts +157 -0
- package/vendor/pi-vcc/src/core/tool-args.ts +14 -0
- package/vendor/pi-vcc/src/details.ts +7 -0
- package/vendor/pi-vcc/src/extract/commits.ts +69 -0
- package/vendor/pi-vcc/src/extract/files.ts +80 -0
- package/vendor/pi-vcc/src/extract/goals.ts +79 -0
- package/vendor/pi-vcc/src/extract/preferences.ts +55 -0
- package/vendor/pi-vcc/src/hooks/before-compact.ts +314 -0
- package/vendor/pi-vcc/src/sections.ts +12 -0
- package/vendor/pi-vcc/src/tools/recall.ts +109 -0
- package/vendor/pi-vcc/src/types.ts +14 -0
- package/vendor/pi-vcc/tests/before-compact-hook.test.ts +204 -0
- package/vendor/pi-vcc/tests/before-compact.test.ts +145 -0
- package/vendor/pi-vcc/tests/brief.test.ts +206 -0
- package/vendor/pi-vcc/tests/build-sections.test.ts +59 -0
- package/vendor/pi-vcc/tests/compile.test.ts +80 -0
- package/vendor/pi-vcc/tests/content.test.ts +31 -0
- package/vendor/pi-vcc/tests/extract-goals.test.ts +86 -0
- package/vendor/pi-vcc/tests/extract-preferences.test.ts +30 -0
- package/vendor/pi-vcc/tests/filter-noise.test.ts +61 -0
- package/vendor/pi-vcc/tests/fixtures.ts +61 -0
- package/vendor/pi-vcc/tests/format-recall.test.ts +30 -0
- package/vendor/pi-vcc/tests/format.test.ts +62 -0
- package/vendor/pi-vcc/tests/lineage.test.ts +33 -0
- package/vendor/pi-vcc/tests/load-messages.test.ts +51 -0
- package/vendor/pi-vcc/tests/normalize.test.ts +97 -0
- package/vendor/pi-vcc/tests/real-sessions.test.ts +38 -0
- package/vendor/pi-vcc/tests/recall-expand.test.ts +15 -0
- package/vendor/pi-vcc/tests/recall-scope.test.ts +32 -0
- package/vendor/pi-vcc/tests/recall-tool-scope.test.ts +67 -0
- package/vendor/pi-vcc/tests/render-entries.test.ts +62 -0
- package/vendor/pi-vcc/tests/report.test.ts +44 -0
- package/vendor/pi-vcc/tests/sanitize.test.ts +24 -0
- package/vendor/pi-vcc/tests/search-entries.test.ts +144 -0
- package/vendor/pi-vcc/tests/support/load-session.ts +23 -0
- package/vendor/pi-vcc/tests/support/real-sessions.ts +51 -0
- package/.pi/pi-vcc-config.json +0 -4
|
@@ -36,7 +36,7 @@ if str(SCRIPT_DIR) not in sys.path:
|
|
|
36
36
|
from harness_web.config import HarnessWebConfig, load_config # noqa: E402
|
|
37
37
|
from harness_web.output import write_search_results # noqa: E402
|
|
38
38
|
from harness_web.scrape import bulk_scrape, map_url, scrape_url # noqa: E402
|
|
39
|
-
from harness_web.
|
|
39
|
+
from harness_web.search import search # noqa: E402
|
|
40
40
|
|
|
41
41
|
DEFAULT_WEB_DIR = ".web"
|
|
42
42
|
|
|
@@ -47,8 +47,8 @@ def _default_out(sub: str) -> Path:
|
|
|
47
47
|
|
|
48
48
|
def cmd_search(args: argparse.Namespace, config: HarnessWebConfig) -> int:
|
|
49
49
|
out = Path(args.output or _default_out("search.json"))
|
|
50
|
-
results =
|
|
51
|
-
write_search_results(out, results, args.query)
|
|
50
|
+
results = search(args.query, limit=args.limit, config=config)
|
|
51
|
+
write_search_results(out, results, args.query, engine=config.search_engine)
|
|
52
52
|
print(f"wrote {out} ({len(results)} results)")
|
|
53
53
|
return 0
|
|
54
54
|
|
|
@@ -76,6 +76,20 @@ def cmd_map(args: argparse.Namespace, config: HarnessWebConfig) -> int:
|
|
|
76
76
|
return 0
|
|
77
77
|
|
|
78
78
|
|
|
79
|
+
def cmd_status(_args: argparse.Namespace, config: HarnessWebConfig) -> int:
|
|
80
|
+
import json
|
|
81
|
+
|
|
82
|
+
payload = {
|
|
83
|
+
"search_engine": config.search_engine,
|
|
84
|
+
"searxng_url": config.searxng_url,
|
|
85
|
+
"fetch_mode": config.fetch_mode,
|
|
86
|
+
"script": str(Path(__file__).resolve()),
|
|
87
|
+
"bootstrap": "ok",
|
|
88
|
+
}
|
|
89
|
+
print(json.dumps(payload, indent=2))
|
|
90
|
+
return 0
|
|
91
|
+
|
|
92
|
+
|
|
79
93
|
def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
|
|
80
94
|
sleep_sec = args.sleep if args.sleep is not None else config.rate_limit_ms / 1000.0
|
|
81
95
|
if args.urls:
|
|
@@ -86,8 +100,8 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
|
|
|
86
100
|
data = json.loads(Path(args.from_search).read_text(encoding="utf-8"))
|
|
87
101
|
urls = [item["url"] for item in data.get("data", {}).get("web", []) if item.get("url")]
|
|
88
102
|
else:
|
|
89
|
-
|
|
90
|
-
urls = [r["url"] for r in
|
|
103
|
+
serp = search(args.query, limit=args.limit, config=config)
|
|
104
|
+
urls = [r["url"] for r in serp]
|
|
91
105
|
|
|
92
106
|
if not urls:
|
|
93
107
|
print("bulk-scrape: no URLs to fetch", file=sys.stderr)
|
|
@@ -111,11 +125,11 @@ def cmd_bulk_scrape(args: argparse.Namespace, config: HarnessWebConfig) -> int:
|
|
|
111
125
|
def build_parser() -> argparse.ArgumentParser:
|
|
112
126
|
p = argparse.ArgumentParser(
|
|
113
127
|
prog="harness-web",
|
|
114
|
-
description="Harness web layer: search (DDG HTML) and scrape (Scrapling).",
|
|
128
|
+
description="Harness web layer: search (DDG HTML or SearXNG) and scrape (Scrapling).",
|
|
115
129
|
)
|
|
116
130
|
sub = p.add_subparsers(dest="command", required=True)
|
|
117
131
|
|
|
118
|
-
ps = sub.add_parser("search", help="Search via
|
|
132
|
+
ps = sub.add_parser("search", help="Search via configured SERP (HARNESS_WEB_SEARCH_ENGINE)")
|
|
119
133
|
ps.add_argument("query", help="Search query")
|
|
120
134
|
ps.add_argument("-o", "--output", help="JSON output path (default: .web/search.json)")
|
|
121
135
|
ps.add_argument("--limit", type=int, default=5)
|
|
@@ -160,6 +174,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
160
174
|
pm.add_argument("--fast", action="store_true")
|
|
161
175
|
pm.set_defaults(func=cmd_map)
|
|
162
176
|
|
|
177
|
+
pst = sub.add_parser("status", help="Print harness-web config as JSON (setup/diagnostics)")
|
|
178
|
+
pst.set_defaults(func=cmd_status)
|
|
179
|
+
|
|
163
180
|
return p
|
|
164
181
|
|
|
165
182
|
|
|
@@ -6,6 +6,8 @@ import os
|
|
|
6
6
|
from dataclasses import dataclass
|
|
7
7
|
from urllib.parse import urlparse
|
|
8
8
|
|
|
9
|
+
SUPPORTED_SEARCH_ENGINES = frozenset({"ddg_html", "searxng"})
|
|
10
|
+
|
|
9
11
|
|
|
10
12
|
def _int_env(name: str, default: int) -> int:
|
|
11
13
|
raw = os.environ.get(name, "").strip()
|
|
@@ -24,6 +26,18 @@ def _fetch_mode() -> str:
|
|
|
24
26
|
return "stealth"
|
|
25
27
|
|
|
26
28
|
|
|
29
|
+
def _normalize_searxng_url(raw: str) -> str:
|
|
30
|
+
url = raw.strip().rstrip("/")
|
|
31
|
+
if not url:
|
|
32
|
+
return ""
|
|
33
|
+
parsed = urlparse(url)
|
|
34
|
+
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
|
35
|
+
raise SystemExit(
|
|
36
|
+
f"Invalid HARNESS_WEB_SEARXNG_URL={raw!r} — expected http(s)://host[:port]"
|
|
37
|
+
)
|
|
38
|
+
return url
|
|
39
|
+
|
|
40
|
+
|
|
27
41
|
_STATIC_HOSTS = frozenset(
|
|
28
42
|
{
|
|
29
43
|
"example.com",
|
|
@@ -50,6 +64,7 @@ def host_is_static(url: str) -> bool:
|
|
|
50
64
|
class HarnessWebConfig:
|
|
51
65
|
fetch_mode: str
|
|
52
66
|
search_engine: str
|
|
67
|
+
searxng_url: str | None
|
|
53
68
|
proxy: str | None
|
|
54
69
|
rate_limit_ms: int
|
|
55
70
|
timeout_ms: int
|
|
@@ -68,13 +83,32 @@ class HarnessWebConfig:
|
|
|
68
83
|
return False
|
|
69
84
|
|
|
70
85
|
|
|
86
|
+
def validate_search_config(config: HarnessWebConfig) -> None:
|
|
87
|
+
engine = config.search_engine
|
|
88
|
+
if engine not in SUPPORTED_SEARCH_ENGINES:
|
|
89
|
+
supported = ", ".join(sorted(SUPPORTED_SEARCH_ENGINES))
|
|
90
|
+
raise SystemExit(
|
|
91
|
+
f"Unsupported HARNESS_WEB_SEARCH_ENGINE={engine!r} (supported: {supported})"
|
|
92
|
+
)
|
|
93
|
+
if engine == "searxng" and not config.searxng_url:
|
|
94
|
+
raise SystemExit(
|
|
95
|
+
"HARNESS_WEB_SEARCH_ENGINE=searxng requires HARNESS_WEB_SEARXNG_URL "
|
|
96
|
+
"(e.g. http://127.0.0.1:8080). Run /harness-setup and choose SearXNG, or set both in .env."
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
71
100
|
def load_config() -> HarnessWebConfig:
|
|
72
101
|
proxy = os.environ.get("HARNESS_WEB_PROXY", "").strip() or None
|
|
73
|
-
|
|
102
|
+
engine = os.environ.get("HARNESS_WEB_SEARCH_ENGINE", "ddg_html").strip() or "ddg_html"
|
|
103
|
+
searx_raw = os.environ.get("HARNESS_WEB_SEARXNG_URL", "").strip()
|
|
104
|
+
searxng_url = _normalize_searxng_url(searx_raw) if searx_raw else None
|
|
105
|
+
config = HarnessWebConfig(
|
|
74
106
|
fetch_mode=_fetch_mode(),
|
|
75
|
-
search_engine=
|
|
76
|
-
|
|
107
|
+
search_engine=engine,
|
|
108
|
+
searxng_url=searxng_url,
|
|
77
109
|
proxy=proxy,
|
|
78
110
|
rate_limit_ms=_int_env("HARNESS_WEB_RATE_LIMIT_MS", 2000),
|
|
79
111
|
timeout_ms=_int_env("HARNESS_WEB_TIMEOUT_MS", 30000),
|
|
80
112
|
)
|
|
113
|
+
validate_search_config(config)
|
|
114
|
+
return config
|
|
@@ -18,13 +18,19 @@ def write_json(path: Path, payload: Any) -> None:
|
|
|
18
18
|
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def write_search_results(
|
|
21
|
+
def write_search_results(
|
|
22
|
+
path: Path,
|
|
23
|
+
results: list[dict[str, str]],
|
|
24
|
+
query: str,
|
|
25
|
+
*,
|
|
26
|
+
engine: str,
|
|
27
|
+
) -> None:
|
|
22
28
|
"""Firecrawl-compatible envelope: data.web[].url|title|description."""
|
|
23
29
|
write_json(
|
|
24
30
|
path,
|
|
25
31
|
{
|
|
26
32
|
"query": query,
|
|
27
|
-
"engine":
|
|
33
|
+
"engine": engine,
|
|
28
34
|
"data": {
|
|
29
35
|
"web": [
|
|
30
36
|
{
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Route harness-web search to the configured SERP backend."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .config import HarnessWebConfig, validate_search_config
|
|
6
|
+
from .search_ddg import search_ddg
|
|
7
|
+
from .search_searxng import search_searxng
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def search(
|
|
11
|
+
query: str,
|
|
12
|
+
*,
|
|
13
|
+
limit: int,
|
|
14
|
+
config: HarnessWebConfig,
|
|
15
|
+
) -> list[dict[str, str]]:
|
|
16
|
+
validate_search_config(config)
|
|
17
|
+
engine = config.search_engine
|
|
18
|
+
if engine == "searxng":
|
|
19
|
+
return search_searxng(query, limit=limit, config=config)
|
|
20
|
+
if engine == "ddg_html":
|
|
21
|
+
return search_ddg(query, limit=limit, config=config)
|
|
22
|
+
raise SystemExit(f"Unsupported HARNESS_WEB_SEARCH_ENGINE={engine!r}")
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
from typing import Any
|
|
5
6
|
from urllib.parse import parse_qs, unquote, urlparse
|
|
6
7
|
|
|
7
8
|
from scrapling.fetchers import Fetcher, StealthyFetcher
|
|
@@ -63,11 +64,6 @@ def search_ddg(
|
|
|
63
64
|
config: HarnessWebConfig,
|
|
64
65
|
impersonate: bool = True,
|
|
65
66
|
) -> list[dict[str, str]]:
|
|
66
|
-
if config.search_engine != "ddg_html":
|
|
67
|
-
raise SystemExit(
|
|
68
|
-
f"Unsupported HARNESS_WEB_SEARCH_ENGINE={config.search_engine!r} (only ddg_html)"
|
|
69
|
-
)
|
|
70
|
-
|
|
71
67
|
kwargs: dict = {
|
|
72
68
|
"params": {"q": query},
|
|
73
69
|
"timeout": config.timeout_sec,
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""SearXNG JSON search API (self-hosted instances)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import ssl
|
|
7
|
+
from typing import Any
|
|
8
|
+
from urllib.error import HTTPError, URLError
|
|
9
|
+
from urllib.parse import urlencode
|
|
10
|
+
from urllib.request import ProxyHandler, Request, build_opener, urlopen
|
|
11
|
+
|
|
12
|
+
from .config import HarnessWebConfig
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _open_url(url: str, *, config: HarnessWebConfig) -> tuple[int, str]:
|
|
16
|
+
req = Request(
|
|
17
|
+
url,
|
|
18
|
+
headers={"Accept": "application/json", "User-Agent": "ultimate-pi-harness-web/1.0"},
|
|
19
|
+
method="GET",
|
|
20
|
+
)
|
|
21
|
+
if config.proxy:
|
|
22
|
+
opener = build_opener(ProxyHandler({"http": config.proxy, "https": config.proxy}))
|
|
23
|
+
resp = opener.open(req, timeout=config.timeout_sec)
|
|
24
|
+
else:
|
|
25
|
+
ctx = ssl.create_default_context()
|
|
26
|
+
resp = urlopen(req, timeout=config.timeout_sec, context=ctx)
|
|
27
|
+
try:
|
|
28
|
+
status = getattr(resp, "status", 200) or 200
|
|
29
|
+
body = resp.read().decode("utf-8", errors="replace")
|
|
30
|
+
return status, body
|
|
31
|
+
finally:
|
|
32
|
+
resp.close()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _parse_results(payload: Any, limit: int) -> list[dict[str, str]]:
|
|
36
|
+
raw = payload.get("results") if isinstance(payload, dict) else None
|
|
37
|
+
if not isinstance(raw, list):
|
|
38
|
+
return []
|
|
39
|
+
out: list[dict[str, str]] = []
|
|
40
|
+
for item in raw:
|
|
41
|
+
if len(out) >= limit:
|
|
42
|
+
break
|
|
43
|
+
if not isinstance(item, dict):
|
|
44
|
+
continue
|
|
45
|
+
url = (item.get("url") or "").strip()
|
|
46
|
+
if not url.startswith("http"):
|
|
47
|
+
continue
|
|
48
|
+
title = (item.get("title") or "").strip()
|
|
49
|
+
description = (item.get("content") or item.get("snippet") or "").strip()
|
|
50
|
+
out.append({"url": url, "title": title, "description": description})
|
|
51
|
+
return out
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def search_searxng(
|
|
55
|
+
query: str,
|
|
56
|
+
*,
|
|
57
|
+
limit: int,
|
|
58
|
+
config: HarnessWebConfig,
|
|
59
|
+
) -> list[dict[str, str]]:
|
|
60
|
+
base = config.searxng_url
|
|
61
|
+
if not base:
|
|
62
|
+
raise SystemExit("HARNESS_WEB_SEARXNG_URL is not set")
|
|
63
|
+
|
|
64
|
+
qs = urlencode({"q": query, "format": "json", "pageno": "1"})
|
|
65
|
+
url = f"{base}/search?{qs}"
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
status, body = _open_url(url, config=config)
|
|
69
|
+
except HTTPError as err:
|
|
70
|
+
status = err.code
|
|
71
|
+
body = err.read().decode("utf-8", errors="replace") if err.fp else ""
|
|
72
|
+
except URLError as err:
|
|
73
|
+
raise SystemExit(
|
|
74
|
+
f"SearXNG request failed ({err.reason}). "
|
|
75
|
+
f"Is the instance running at {base}? "
|
|
76
|
+
"Run: node \"$UP_PKG/.pi/scripts/harness-searxng-bootstrap.mjs\""
|
|
77
|
+
) from err
|
|
78
|
+
|
|
79
|
+
if status == 403:
|
|
80
|
+
raise SystemExit(
|
|
81
|
+
"SearXNG returned 403 for format=json. Enable json under search.formats "
|
|
82
|
+
"in settings.yml (see .searxng/core-config/settings.yml or SearXNG docs)."
|
|
83
|
+
)
|
|
84
|
+
if status != 200:
|
|
85
|
+
snippet = body[:200].replace("\n", " ")
|
|
86
|
+
raise SystemExit(f"SearXNG search failed (HTTP {status}): {snippet}")
|
|
87
|
+
|
|
88
|
+
try:
|
|
89
|
+
payload = json.loads(body)
|
|
90
|
+
except json.JSONDecodeError as err:
|
|
91
|
+
raise SystemExit(f"SearXNG returned non-JSON response from {url}") from err
|
|
92
|
+
|
|
93
|
+
results = _parse_results(payload, limit)
|
|
94
|
+
if not results and isinstance(payload, dict):
|
|
95
|
+
unresponsive = payload.get("unresponsive_engines")
|
|
96
|
+
if unresponsive:
|
|
97
|
+
raise SystemExit(
|
|
98
|
+
f"SearXNG returned no results; upstream engines unresponsive: {unresponsive}"
|
|
99
|
+
)
|
|
100
|
+
return results
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ultimate-pi harness settings (env-only). Re-exported for vendored pi-vcc layout.
|
|
3
|
+
*/
|
|
4
|
+
export type { PiVccSettings } from "../../../../.pi/extensions/lib/harness-vcc-settings.js";
|
|
5
|
+
export {
|
|
6
|
+
loadSettings,
|
|
7
|
+
scaffoldSettings,
|
|
8
|
+
} from "../../../../.pi/extensions/lib/harness-vcc-settings.js";
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Re-fetch upstream pi-vcc and re-apply ultimate-pi patches.
|
|
3
|
+
set -euo pipefail
|
|
4
|
+
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
5
|
+
VEND="$ROOT/vendor/pi-vcc"
|
|
6
|
+
SETTINGS_STUB="$ROOT/.pi/scripts/vendor-pi-vcc-settings.stub.ts"
|
|
7
|
+
|
|
8
|
+
rm -rf "$VEND"
|
|
9
|
+
git clone --depth 1 https://github.com/sting8k/pi-vcc.git "$VEND"
|
|
10
|
+
COMMIT="$(git -C "$VEND" rev-parse HEAD)"
|
|
11
|
+
rm -rf "$VEND/.git"
|
|
12
|
+
|
|
13
|
+
# Env-only settings (no JSON config files)
|
|
14
|
+
cp "$SETTINGS_STUB" "$VEND/src/core/settings.ts"
|
|
15
|
+
|
|
16
|
+
# Drop config scaffolding on extension load
|
|
17
|
+
sed -i '/scaffoldSettings/d' "$VEND/index.ts"
|
|
18
|
+
sed -i '/import.*scaffoldSettings/d' "$VEND/index.ts"
|
|
19
|
+
|
|
20
|
+
# Telemetry / compaction details label
|
|
21
|
+
sed -i 's/compactor: "pi-vcc";/compactor: "pi-vcc" | "ultimate-pi-vcc";/' "$VEND/src/details.ts"
|
|
22
|
+
sed -i 's/compactor: "pi-vcc"/compactor: "ultimate-pi-vcc"/' "$VEND/src/hooks/before-compact.ts"
|
|
23
|
+
|
|
24
|
+
cat >"$VEND/UPSTREAM_PIN.md" <<EOF
|
|
25
|
+
# Vendored \`pi-vcc\`
|
|
26
|
+
|
|
27
|
+
- **Repository:** https://github.com/sting8k/pi-vcc
|
|
28
|
+
- **Conceptual basis:** [lllyasviel/VCC](https://github.com/lllyasviel/VCC) (View-oriented Conversation Compiler)
|
|
29
|
+
- **License:** MIT (see upstream repository)
|
|
30
|
+
- **Pinned upstream commit:** \`$COMMIT\`
|
|
31
|
+
- **Local changes:**
|
|
32
|
+
- \`src/core/settings.ts\` re-exports env-only [\`harness-vcc-settings\`](../../.pi/extensions/lib/harness-vcc-settings.ts) (\`HARNESS_VCC_COMPACTION\`, \`HARNESS_VCC_DEBUG\`)
|
|
33
|
+
- No \`scaffoldSettings\` / no \`PI_VCC_CONFIG_PATH\`
|
|
34
|
+
- Compaction \`details.compactor\` is \`ultimate-pi-vcc\`
|
|
35
|
+
|
|
36
|
+
**Refresh upstream:** run \`npm run vendor:sync-vcc\` from ultimate-pi root.
|
|
37
|
+
EOF
|
|
38
|
+
|
|
39
|
+
rm -f "$VEND/package-lock.json"
|
|
40
|
+
echo "✓ Vendor refreshed at $VEND (commit $COMMIT)"
|
package/CHANGELOG.md
CHANGED
|
@@ -2,17 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project are documented in this file.
|
|
4
4
|
|
|
5
|
-
## [
|
|
5
|
+
## [Unreleased]
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
## [v0.5.0] — 2026-05-17
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
### ✨ Features
|
|
10
|
+
|
|
11
|
+
- **web_search / web_fetch pi tools:** wrap `harness-web.py` with session injection and a bash guard so agents skip `UP_PKG` and scrapling import preflights.
|
|
12
|
+
- **SearXNG search backend:** pluggable `HARNESS_WEB_SEARCH_ENGINE` (`ddg_html` | `searxng`) with Docker bootstrap via `harness-searxng-bootstrap.mjs`.
|
|
13
|
+
- **harness-web status:** JSON config subcommand for setup and diagnostics.
|
|
14
|
+
|
|
15
|
+
### 🔧 Chores
|
|
16
|
+
|
|
17
|
+
- Apply pre-commit format and refresh `graphify-out` after harness-web tools merge.
|
|
18
|
+
|
|
19
|
+
## [v0.4.1] — 2026-05-17
|
|
11
20
|
|
|
12
21
|
### ✨ Features
|
|
13
22
|
|
|
14
|
-
- **
|
|
15
|
-
- **Harness subagents
|
|
23
|
+
- **In-house VCC compaction:** vendored [pi-vcc](https://github.com/sting8k/pi-vcc) (inspired by [lllyasviel/VCC](https://github.com/lllyasviel/VCC)); removed `@sting8k/pi-vcc` npm dependency. Configuration is **env-only** (`HARNESS_VCC_COMPACTION`, `HARNESS_VCC_DEBUG`) — no `PI_VCC_CONFIG_PATH` or JSON config files. VCC overrides `/compact` and auto-compaction by default; set `HARNESS_VCC_COMPACTION=false` for Pi LLM compaction. Refresh upstream: `npm run vendor:sync-vcc`.
|
|
24
|
+
- **Harness subagents:** vendored harness-subagents extension with Sentrux bootstrap agent and related harness wiring.
|
|
25
|
+
- **Harness web:** replace Firecrawl with Scrapling-based `harness-web` CLI for search and fetch.
|
|
26
|
+
|
|
27
|
+
### 🔧 Chores
|
|
28
|
+
|
|
29
|
+
- Format `.pi/settings.json` / `.pi/settings.example.json` and refresh `graphify-out` after VCC merge.
|
|
16
30
|
|
|
17
31
|
## [v0.3.1] — 2026-05-15
|
|
18
32
|
|
package/THIRD_PARTY_NOTICES.md
CHANGED
|
@@ -1,30 +1,16 @@
|
|
|
1
1
|
# Third-party notices
|
|
2
2
|
|
|
3
|
-
## Design references (not vendored)
|
|
4
|
-
|
|
5
|
-
The in-house `ask_user` harness extension (`.pi/extensions/harness-ask-user.ts`) borrows **behavior and UX patterns only** from these projects. No code from them is bundled.
|
|
6
|
-
|
|
7
|
-
| Project | License | Reference |
|
|
8
|
-
|---------|---------|-----------|
|
|
9
|
-
| [pi-ask-user](https://github.com/edlsh/pi-ask-user) | MIT | Overlay/inline modes, SelectList + Editor, custom renderCall/renderResult, headless fallback |
|
|
10
|
-
| [@pi-unipi/ask-user](https://cdn.jsdelivr.net/npm/@pi-unipi/unipi@2.0.1/packages/ask-user/README.md) | (package docs) | Tool contract: `question`, `context`, `options`, `allowMultiple`, `allowFreeform` |
|
|
11
|
-
| [rpiv-ask-user-question](https://github.com/juicesharp/rpiv-mono/tree/main/packages/rpiv-ask-user-question) | MIT | Rich option rows, decision-handshake policy text, structured answer envelope |
|
|
12
|
-
|
|
13
|
-
## @tintinweb/pi-subagents (vendored in harness-subagents)
|
|
14
|
-
|
|
15
|
-
- **Project:** https://github.com/tintinweb/pi-subagents
|
|
16
|
-
- **Version pinned:** 0.7.3 (source vendored under `.pi/extensions/lib/harness-subagents/vendored/`)
|
|
17
|
-
- **License:** MIT
|
|
18
|
-
- **Notes:** Adapted for `@mariozechner/pi-coding-agent` in ultimate-pi `harness-subagents` extension. Agent discovery replaced with package-root recursive loader.
|
|
19
|
-
|
|
20
|
-
## subagent-v2 reference (design only)
|
|
21
|
-
|
|
22
|
-
- **Path:** `raw/references/subagents/extensions/subagent-v2/`
|
|
23
|
-
- **License:** Treat as third-party reference; not shipped in npm `files`. Blackboard, supervisor, and isolated session patterns were ported into `harness-subagents` without copying the earendil-works Pi stack.
|
|
24
|
-
|
|
25
3
|
## pi-model-router (vendored)
|
|
26
4
|
|
|
27
5
|
- **Project:** https://github.com/yeliu84/pi-model-router
|
|
28
6
|
- **License:** MIT ([vendor/pi-model-router/LICENSE](vendor/pi-model-router/LICENSE))
|
|
29
7
|
- **Pinned revision:** See [vendor/pi-model-router/UPSTREAM_PIN.md](vendor/pi-model-router/UPSTREAM_PIN.md)
|
|
30
8
|
- ultimate-pi loads it from [`vendor/pi-model-router`](vendor/pi-model-router); import specifiers were adapted for `@mariozechner/pi-coding-agent` and related Pi packages.
|
|
9
|
+
|
|
10
|
+
## pi-vcc (vendored)
|
|
11
|
+
|
|
12
|
+
- **Project:** https://github.com/sting8k/pi-vcc
|
|
13
|
+
- **Conceptual basis:** https://github.com/lllyasviel/VCC (View-oriented Conversation Compiler)
|
|
14
|
+
- **License:** MIT (see upstream repository)
|
|
15
|
+
- **Pinned revision:** See [vendor/pi-vcc/UPSTREAM_PIN.md](vendor/pi-vcc/UPSTREAM_PIN.md)
|
|
16
|
+
- ultimate-pi loads it from [`vendor/pi-vcc`](vendor/pi-vcc) via [`.pi/extensions/ultimate-pi-vcc.ts`](.pi/extensions/ultimate-pi-vcc.ts). Harness configuration is env-only: `HARNESS_VCC_COMPACTION`, `HARNESS_VCC_DEBUG` ([`.pi/extensions/lib/harness-vcc-settings.ts`](.pi/extensions/lib/harness-vcc-settings.ts)). Maintainer refresh: `npm run vendor:sync-vcc`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ultimate-pi",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Ultimate AI coding harness for pi.dev — extensible skills, Obsidian wiki knowledge layer, compressed context, deterministic output",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
@@ -60,7 +60,6 @@
|
|
|
60
60
|
".pi/settings.example.json",
|
|
61
61
|
".pi/auto-commit.json",
|
|
62
62
|
".pi/mcp.json",
|
|
63
|
-
".pi/pi-vcc-config.json",
|
|
64
63
|
".pi/SYSTEM.md",
|
|
65
64
|
".pi/PACKAGING.md",
|
|
66
65
|
"AGENTS.md",
|
|
@@ -68,21 +67,24 @@
|
|
|
68
67
|
"CHANGELOG.md",
|
|
69
68
|
"README.md",
|
|
70
69
|
"THIRD_PARTY_NOTICES.md",
|
|
71
|
-
"vendor/pi-model-router"
|
|
70
|
+
"vendor/pi-model-router",
|
|
71
|
+
"vendor/pi-vcc"
|
|
72
72
|
],
|
|
73
73
|
"peerDependencies": {
|
|
74
74
|
"@mariozechner/pi-coding-agent": "*"
|
|
75
75
|
},
|
|
76
76
|
"scripts": {
|
|
77
|
-
"check:ts": "tsc --noEmit --target ES2023 --lib ES2023 --moduleResolution nodenext --module nodenext --skipLibCheck .pi/extensions/dotenv-loader.ts .pi/extensions/lib/posthog-node.d.ts .pi/extensions/lib/harness-posthog.ts .pi/extensions/lib/harness-paths.ts .pi/extensions/pi-model-router-harness.ts .pi/extensions/provider-payload-sanitize.ts .pi/extensions/harness-telemetry.ts .pi/extensions/harness-ask-user.ts .pi/extensions/lib/ask-user/schema.ts .pi/extensions/lib/ask-user/types.ts .pi/extensions/lib/ask-user/validate.ts .pi/extensions/lib/ask-user/dialog.ts .pi/extensions/lib/ask-user/fallback.ts .pi/extensions/lib/ask-user/render.ts .pi/extensions/trace-recorder.ts .pi/extensions/observation-bus.ts .pi/extensions/drift-monitor.ts .pi/extensions/sentrux-rules-sync.ts .pi/extensions/custom-header.ts .pi/extensions/lib/harness-subagents/agent-loader.ts .pi/extensions/lib/harness-subagents/agent-parser.ts .pi/extensions/lib/harness-subagents/agent-manifest.ts .pi/extensions/lib/harness-subagents/blackboard.ts .pi/extensions/lib/harness-subagents/blackboard-tool.ts .pi/extensions/lib/harness-subagents/spawn-policy.ts .pi/extensions/lib/harness-subagents/types-blackboard.ts",
|
|
77
|
+
"check:ts": "tsc --noEmit --target ES2023 --lib ES2023 --moduleResolution nodenext --module nodenext --skipLibCheck .pi/extensions/lib/harness-vcc-settings.ts .pi/extensions/dotenv-loader.ts .pi/extensions/lib/posthog-node.d.ts .pi/extensions/lib/harness-posthog.ts .pi/extensions/lib/harness-paths.ts .pi/extensions/pi-model-router-harness.ts .pi/extensions/provider-payload-sanitize.ts .pi/extensions/harness-telemetry.ts .pi/extensions/harness-ask-user.ts .pi/extensions/lib/ask-user/schema.ts .pi/extensions/lib/ask-user/types.ts .pi/extensions/lib/ask-user/validate.ts .pi/extensions/lib/ask-user/dialog.ts .pi/extensions/lib/ask-user/fallback.ts .pi/extensions/lib/ask-user/render.ts .pi/extensions/trace-recorder.ts .pi/extensions/observation-bus.ts .pi/extensions/drift-monitor.ts .pi/extensions/sentrux-rules-sync.ts .pi/extensions/custom-header.ts .pi/extensions/lib/harness-subagents/agent-loader.ts .pi/extensions/lib/harness-subagents/agent-parser.ts .pi/extensions/lib/harness-subagents/agent-manifest.ts .pi/extensions/lib/harness-subagents/blackboard.ts .pi/extensions/lib/harness-subagents/blackboard-tool.ts .pi/extensions/lib/harness-subagents/spawn-policy.ts .pi/extensions/lib/harness-subagents/types-blackboard.ts .pi/extensions/harness-web-tools.ts .pi/extensions/harness-web-guard.ts .pi/extensions/lib/harness-web/run-cli.ts",
|
|
78
78
|
"vendor:sync-router": "bash .pi/scripts/vendor-sync-pi-model-router.sh",
|
|
79
|
+
"vendor:sync-vcc": "bash .pi/scripts/vendor-sync-pi-vcc.sh",
|
|
79
80
|
"release": "bash .pi/scripts/release.sh",
|
|
80
81
|
"lint": "biome check",
|
|
81
82
|
"lint:fix": "biome check --fix",
|
|
82
83
|
"format": "biome format --write",
|
|
83
84
|
"format:check": "biome format",
|
|
84
85
|
"prepare": "lefthook install",
|
|
85
|
-
"test": "node --test test/harness-verify.test.mjs test/harness-ask-user.test.mjs test/harness-subagents-loader.test.mjs test/sentrux-rules-sync.test.mjs",
|
|
86
|
+
"test": "node --test test/harness-verify.test.mjs test/harness-ask-user.test.mjs test/harness-subagents-loader.test.mjs test/sentrux-rules-sync.test.mjs && npx -y tsx --test test/harness-vcc-settings.test.ts",
|
|
87
|
+
"test:vcc": "npx -y tsx --test vendor/pi-vcc/tests/*.test.ts",
|
|
86
88
|
"harness:sentrux-bootstrap": "node .pi/scripts/harness-sentrux-bootstrap.mjs",
|
|
87
89
|
"harness:sentrux-sync": "node .pi/scripts/sentrux-rules-sync.mjs --force",
|
|
88
90
|
"test:integration": "npx -y tsx --test test/cursor-sdk-provider.integration.test.ts"
|
|
@@ -99,7 +101,6 @@
|
|
|
99
101
|
},
|
|
100
102
|
"dependencies": {
|
|
101
103
|
"@posthog/pi": "latest",
|
|
102
|
-
"@sting8k/pi-vcc": "^0.3.12",
|
|
103
104
|
"asciify-image": "^0.1.10",
|
|
104
105
|
"croner": "^9.0.0",
|
|
105
106
|
"jimp": "^1.6.1",
|