pi-smart-compact 9.6.0 → 9.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +28 -14
- package/CHANGELOG.md +46 -0
- package/README.md +30 -10
- package/dist/app/mode-policy.d.ts +3 -1
- package/dist/app/mode-policy.d.ts.map +1 -1
- package/dist/app/preflight.d.ts.map +1 -1
- package/dist/app/register-smart-compact-tool.d.ts.map +1 -1
- package/dist/app/run-context.d.ts +3 -3
- package/dist/app/run-context.d.ts.map +1 -1
- package/dist/app/run-smart-compact.d.ts.map +1 -1
- package/dist/app/steps/extract.d.ts.map +1 -1
- package/dist/app/steps/prepare.d.ts.map +1 -1
- package/dist/app/steps/state.d.ts.map +1 -1
- package/dist/app/steps/synthesize.d.ts.map +1 -1
- package/dist/app/steps/tier.d.ts +5 -4
- package/dist/app/steps/tier.d.ts.map +1 -1
- package/dist/app/steps/window.d.ts.map +1 -1
- package/dist/constants.d.ts +1 -1
- package/dist/domain/provider-evaluation.d.ts.map +1 -1
- package/dist/domain/telemetry.d.ts.map +1 -1
- package/dist/domain/tool-semantics.d.ts.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1489 -1248
- package/dist/infra/ai-messages.d.ts +6 -2
- package/dist/infra/ai-messages.d.ts.map +1 -1
- package/dist/phases/explore.d.ts.map +1 -1
- package/dist/phases/synthesize.d.ts +5 -0
- package/dist/phases/synthesize.d.ts.map +1 -1
- package/dist/phases/verify.d.ts.map +1 -1
- package/dist/provider-eval.js +348 -22
- package/dist/provider-scenario-eval.js +363 -33
- package/dist/telemetry-report.js +125 -22
- package/dist/types.d.ts +3 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/ui/error-format.d.ts +3 -1
- package/dist/ui/error-format.d.ts.map +1 -1
- package/dist/ui/metrics-report.d.ts.map +1 -1
- package/dist/ui/overlays.d.ts.map +1 -1
- package/dist/utils/cache.d.ts.map +1 -1
- package/dist/utils/file-needles.d.ts +24 -0
- package/dist/utils/file-needles.d.ts.map +1 -1
- package/dist/utils/file-ref-detect.d.ts +6 -1
- package/dist/utils/file-ref-detect.d.ts.map +1 -1
- package/dist/utils/helpers.d.ts +1 -0
- package/dist/utils/helpers.d.ts.map +1 -1
- package/dist/utils/session-log.d.ts.map +1 -1
- package/package.json +4 -2
|
@@ -5,7 +5,7 @@ var __require = import.meta.require;
|
|
|
5
5
|
import { ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent";
|
|
6
6
|
|
|
7
7
|
// src/constants.ts
|
|
8
|
-
var VERSION = "9.6.
|
|
8
|
+
var VERSION = "9.6.2";
|
|
9
9
|
var SETTLED_TRIGGER_COOLDOWN_MS = 10 * 60000;
|
|
10
10
|
var COMPACT_SYSTEM_PREFIX = "You are an expert conversation summarizer for a coding agent. " + "Produce structured markdown summaries. " + "Follow output format exactly. " + "Use EXACT names \u2014 never paraphrase code identifiers. " + "Trust deterministic extraction data over intuition.";
|
|
11
11
|
var PROFILES = {
|
|
@@ -690,38 +690,117 @@ function buildPathNeedles(filePath) {
|
|
|
690
690
|
}
|
|
691
691
|
return needles;
|
|
692
692
|
}
|
|
693
|
-
|
|
694
|
-
|
|
693
|
+
var MAX_INDEXED_SUFFIX_CHARS = 1024;
|
|
694
|
+
function buildPathNeedleOwnershipIndex(allPaths) {
|
|
695
|
+
const owners = new Map;
|
|
696
|
+
const normalizedPaths = allPaths.map(normalizePath);
|
|
697
|
+
let hasUnindexedSuffixes = false;
|
|
698
|
+
for (const normalized of normalizedPaths) {
|
|
699
|
+
const suffixes = new Set([normalized]);
|
|
700
|
+
for (let index = 0;index < normalized.length; index++) {
|
|
701
|
+
if (normalized[index] !== "/")
|
|
702
|
+
continue;
|
|
703
|
+
if (normalized.length - index - 1 <= MAX_INDEXED_SUFFIX_CHARS) {
|
|
704
|
+
suffixes.add(normalized.slice(index + 1));
|
|
705
|
+
} else {
|
|
706
|
+
hasUnindexedSuffixes = true;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
for (const suffix of suffixes) {
|
|
710
|
+
owners.set(suffix, (owners.get(suffix) ?? 0) + 1);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return { counts: owners, normalizedPaths, hasUnindexedSuffixes };
|
|
714
|
+
}
|
|
715
|
+
function buildUniquePathNeedlesFromIndex(filePath, owners) {
|
|
695
716
|
return buildPathNeedles(filePath).filter((needle) => {
|
|
696
|
-
|
|
697
|
-
|
|
717
|
+
if (!owners.hasUnindexedSuffixes)
|
|
718
|
+
return owners.counts.get(needle) === 1;
|
|
719
|
+
let count = 0;
|
|
720
|
+
for (const candidate of owners.normalizedPaths) {
|
|
698
721
|
if (candidate === needle || candidate.endsWith("/" + needle))
|
|
699
|
-
|
|
700
|
-
if (
|
|
722
|
+
count++;
|
|
723
|
+
if (count > 1)
|
|
701
724
|
return false;
|
|
702
725
|
}
|
|
703
|
-
return
|
|
726
|
+
return count === 1;
|
|
704
727
|
});
|
|
705
728
|
}
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
const
|
|
711
|
-
|
|
729
|
+
var PATH_CANDIDATE_CHAR_RE = /[\w./-]/;
|
|
730
|
+
function buildKnownPathReferenceIndex(knownPaths) {
|
|
731
|
+
const segmentSuffixes = new Set;
|
|
732
|
+
const boundarySuffixes = new Set;
|
|
733
|
+
const normalizedPaths = [];
|
|
734
|
+
let hasUnindexedSuffixes = false;
|
|
735
|
+
for (const path of knownPaths) {
|
|
712
736
|
const normalizedPath = normalizePath(path).replace(/^\/+/, "");
|
|
737
|
+
if (!normalizedPath)
|
|
738
|
+
continue;
|
|
739
|
+
normalizedPaths.push(normalizedPath);
|
|
740
|
+
segmentSuffixes.add(normalizedPath);
|
|
741
|
+
for (let index = 0;index < normalizedPath.length; index++) {
|
|
742
|
+
if (normalizedPath[index] === "/") {
|
|
743
|
+
if (normalizedPath.length - index - 1 <= MAX_INDEXED_SUFFIX_CHARS) {
|
|
744
|
+
segmentSuffixes.add(normalizedPath.slice(index + 1));
|
|
745
|
+
} else {
|
|
746
|
+
hasUnindexedSuffixes = true;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
if (index > 0 && !PATH_CANDIDATE_CHAR_RE.test(normalizedPath[index - 1])) {
|
|
750
|
+
if (normalizedPath.length - index <= MAX_INDEXED_SUFFIX_CHARS) {
|
|
751
|
+
boundarySuffixes.add(normalizedPath.slice(index));
|
|
752
|
+
} else {
|
|
753
|
+
hasUnindexedSuffixes = true;
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
return {
|
|
759
|
+
segmentSuffixes,
|
|
760
|
+
sortedSegmentSuffixes: [...segmentSuffixes].sort(),
|
|
761
|
+
boundarySuffixes,
|
|
762
|
+
normalizedPaths,
|
|
763
|
+
hasUnindexedSuffixes
|
|
764
|
+
};
|
|
765
|
+
}
|
|
766
|
+
function matchesKnownPathReference(normalizedRef, normalizedPaths) {
|
|
767
|
+
const pathShaped = normalizedRef.includes("/");
|
|
768
|
+
return normalizedPaths.some((normalizedPath) => {
|
|
713
769
|
if (normalizedPath === normalizedRef || normalizedPath.endsWith("/" + normalizedRef))
|
|
714
770
|
return true;
|
|
715
771
|
if (normalizedPath.endsWith(normalizedRef)) {
|
|
716
772
|
const boundary = normalizedPath[normalizedPath.length - normalizedRef.length - 1];
|
|
717
|
-
if (boundary &&
|
|
773
|
+
if (boundary && !PATH_CANDIDATE_CHAR_RE.test(boundary))
|
|
718
774
|
return true;
|
|
719
775
|
}
|
|
720
776
|
if (!pathShaped)
|
|
721
777
|
return false;
|
|
722
|
-
return normalizedPath.
|
|
778
|
+
return normalizedPath.startsWith(normalizedRef + "/") || normalizedPath.includes("/" + normalizedRef + "/");
|
|
723
779
|
});
|
|
724
780
|
}
|
|
781
|
+
function sortedHasPrefix(values, prefix) {
|
|
782
|
+
let low = 0;
|
|
783
|
+
let high = values.length;
|
|
784
|
+
while (low < high) {
|
|
785
|
+
const middle = low + high >>> 1;
|
|
786
|
+
if (values[middle] < prefix)
|
|
787
|
+
low = middle + 1;
|
|
788
|
+
else
|
|
789
|
+
high = middle;
|
|
790
|
+
}
|
|
791
|
+
return values[low]?.startsWith(prefix) ?? false;
|
|
792
|
+
}
|
|
793
|
+
function isKnownPathReferenceInIndex(ref, index) {
|
|
794
|
+
const normalizedRef = normalizePath(ref).replace(/^\/+/, "");
|
|
795
|
+
if (!normalizedRef)
|
|
796
|
+
return false;
|
|
797
|
+
if (index.segmentSuffixes.has(normalizedRef) || index.boundarySuffixes.has(normalizedRef)) {
|
|
798
|
+
return true;
|
|
799
|
+
}
|
|
800
|
+
if (normalizedRef.includes("/") && sortedHasPrefix(index.sortedSegmentSuffixes, normalizedRef + "/"))
|
|
801
|
+
return true;
|
|
802
|
+
return index.hasUnindexedSuffixes ? matchesKnownPathReference(normalizedRef, index.normalizedPaths) : false;
|
|
803
|
+
}
|
|
725
804
|
|
|
726
805
|
// src/domain/tool-semantics.ts
|
|
727
806
|
var PATH_KEYS = [
|
|
@@ -855,7 +934,12 @@ function damageReportsFile() {
|
|
|
855
934
|
// src/utils/file-ref-detect.ts
|
|
856
935
|
var CODE_EXT_RE = /\.(ts|tsx|js|jsx|mjs|cjs|rs|py|go|java|rb|cs|cpp|c|h|hpp|swift|kt|scala|php|css|scss|html|json|yaml|yml|toml|md|mdx|sh|sql|tf|ini|env|lock|gradle|xml)$/i;
|
|
857
936
|
var VERSION_RE = /^v?\d+(?:\.\d+)+(?:[-+][\w.-]+)?$/i;
|
|
858
|
-
|
|
937
|
+
function isAsciiWordCode(code) {
|
|
938
|
+
return code >= 48 && code <= 57 || code >= 65 && code <= 90 || code === 95 || code >= 97 && code <= 122;
|
|
939
|
+
}
|
|
940
|
+
function isCandidateCode(code) {
|
|
941
|
+
return isAsciiWordCode(code) || code === 45 || code === 46 || code === 47;
|
|
942
|
+
}
|
|
859
943
|
function isLikelyFileRef(candidate) {
|
|
860
944
|
if (candidate.startsWith("//") || VERSION_RE.test(candidate))
|
|
861
945
|
return false;
|
|
@@ -866,13 +950,32 @@ function isLikelyFileRef(candidate) {
|
|
|
866
950
|
return CODE_EXT_RE.test(candidate);
|
|
867
951
|
}
|
|
868
952
|
function extractFileRefs(summary) {
|
|
869
|
-
const matcher = new RegExp(FILE_REF_CANDIDATE_RE.source, FILE_REF_CANDIDATE_RE.flags);
|
|
870
953
|
const refs = [];
|
|
871
|
-
|
|
872
|
-
|
|
954
|
+
let cursor = 0;
|
|
955
|
+
while (cursor < summary.length) {
|
|
956
|
+
while (cursor < summary.length && !isCandidateCode(summary.charCodeAt(cursor)))
|
|
957
|
+
cursor++;
|
|
958
|
+
const runStart = cursor;
|
|
959
|
+
while (cursor < summary.length && isCandidateCode(summary.charCodeAt(cursor)))
|
|
960
|
+
cursor++;
|
|
961
|
+
const runEnd = cursor;
|
|
962
|
+
let extensionDot = -1;
|
|
963
|
+
for (let index = runStart + 1;index + 1 < runEnd; index++) {
|
|
964
|
+
if (summary.charCodeAt(index) === 46 && isAsciiWordCode(summary.charCodeAt(index + 1))) {
|
|
965
|
+
extensionDot = index;
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
if (extensionDot < 0)
|
|
873
969
|
continue;
|
|
874
|
-
|
|
875
|
-
|
|
970
|
+
let matchEnd = extensionDot + 2;
|
|
971
|
+
while (matchEnd < runEnd && isAsciiWordCode(summary.charCodeAt(matchEnd))) {
|
|
972
|
+
matchEnd++;
|
|
973
|
+
}
|
|
974
|
+
const candidate = summary.slice(runStart, matchEnd);
|
|
975
|
+
if (/[\\/]/.test(summary[matchEnd] ?? ""))
|
|
976
|
+
continue;
|
|
977
|
+
if (isLikelyFileRef(candidate))
|
|
978
|
+
refs.push(candidate);
|
|
876
979
|
}
|
|
877
980
|
return refs;
|
|
878
981
|
}
|
|
@@ -1564,6 +1667,229 @@ var processToolSupport = new ToolSupportCache;
|
|
|
1564
1667
|
var processTokenCalibration = new TokenCalibrationStore;
|
|
1565
1668
|
var _default = createServices();
|
|
1566
1669
|
|
|
1670
|
+
// src/domain/telemetry.ts
|
|
1671
|
+
function p95(values) {
|
|
1672
|
+
if (!values.length)
|
|
1673
|
+
return 0;
|
|
1674
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
1675
|
+
return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0;
|
|
1676
|
+
}
|
|
1677
|
+
function stats(entries, damage) {
|
|
1678
|
+
const evidence = entries.filter((entry) => entry.status !== "dry-run");
|
|
1679
|
+
const successfulRuns = evidence.filter((entry) => entry.status === "success");
|
|
1680
|
+
const quality = successfulRuns.filter((entry) => typeof entry.verificationScore === "number");
|
|
1681
|
+
const appliedRunIds = new Set(successfulRuns.filter((entry) => typeof entry.runId === "string" && entry.runId.length >= 8).map((entry) => entry.runId));
|
|
1682
|
+
const observedScores = new Map;
|
|
1683
|
+
for (const observation of damage) {
|
|
1684
|
+
if (!observation.runId || !appliedRunIds.has(observation.runId) || typeof observation.damageScore !== "number" || !Number.isFinite(observation.damageScore))
|
|
1685
|
+
continue;
|
|
1686
|
+
observedScores.set(observation.runId, Math.max(observedScores.get(observation.runId) ?? 0, Math.max(0, Math.min(100, observation.damageScore))));
|
|
1687
|
+
}
|
|
1688
|
+
const damaging = [...observedScores.values()].filter((score) => score > 0).length;
|
|
1689
|
+
return {
|
|
1690
|
+
runs: entries.length,
|
|
1691
|
+
appliedRuns: evidence.length,
|
|
1692
|
+
successRate: evidence.length ? successfulRuns.length / evidence.length : 1,
|
|
1693
|
+
avgQuality: quality.length ? quality.reduce((sum, entry) => sum + (entry.verificationScore ?? 0), 0) / quality.length : null,
|
|
1694
|
+
qualityCoverage: evidence.length ? quality.length / evidence.length : 0,
|
|
1695
|
+
p95LatencyMs: p95(evidence.map((entry) => entry.durationMs ?? entry.avgLatency).filter(Number.isFinite)),
|
|
1696
|
+
avgTokens: evidence.length ? evidence.reduce((sum, entry) => sum + entry.totalInput + entry.totalCacheHit + (entry.totalCacheWrite ?? 0) + entry.totalOutput, 0) / evidence.length : 0,
|
|
1697
|
+
fallbackRate: evidence.length ? evidence.filter((entry) => entry.method === "heuristic" || Array.isArray(entry.providerRoutes) && entry.providerRoutes.some((route) => route.successes < route.calls)).length / evidence.length : 0,
|
|
1698
|
+
damageRate: observedScores.size ? damaging / observedScores.size : 0,
|
|
1699
|
+
damageCoverage: successfulRuns.length ? observedScores.size / successfulRuns.length : 0
|
|
1700
|
+
};
|
|
1701
|
+
}
|
|
1702
|
+
function roundStats(value) {
|
|
1703
|
+
return {
|
|
1704
|
+
...value,
|
|
1705
|
+
successRate: Math.round(value.successRate * 1000) / 1000,
|
|
1706
|
+
avgQuality: value.avgQuality == null ? null : Math.round(value.avgQuality * 10) / 10,
|
|
1707
|
+
qualityCoverage: Math.round(value.qualityCoverage * 1000) / 1000,
|
|
1708
|
+
p95LatencyMs: Math.round(value.p95LatencyMs),
|
|
1709
|
+
avgTokens: Math.round(value.avgTokens),
|
|
1710
|
+
fallbackRate: Math.round(value.fallbackRate * 1000) / 1000,
|
|
1711
|
+
damageRate: Math.round(value.damageRate * 1000) / 1000,
|
|
1712
|
+
damageCoverage: Math.round(value.damageCoverage * 1000) / 1000
|
|
1713
|
+
};
|
|
1714
|
+
}
|
|
1715
|
+
function assessCanary(entries, damageEntries, options) {
|
|
1716
|
+
const minCanaryRuns = Math.max(5, options.minCanaryRuns ?? 20);
|
|
1717
|
+
const canaryEntries = entries.filter((entry) => entry.metricsSchemaVersion === 2 && entry.version === options.version && entry.releaseChannel === "canary").slice(-Math.max(100, minCanaryRuns));
|
|
1718
|
+
const baselineEntries = entries.filter((entry) => entry.metricsSchemaVersion === 2 && (entry.releaseChannel ?? "stable") === "stable").slice(-(options.baselineRuns ?? Math.max(50, minCanaryRuns * 2)));
|
|
1719
|
+
const baseline = stats(baselineEntries, damageEntries);
|
|
1720
|
+
const canary = stats(canaryEntries, damageEntries);
|
|
1721
|
+
const triggers = [];
|
|
1722
|
+
const failureBaseline = 1 - baseline.successRate;
|
|
1723
|
+
const failureCanary = 1 - canary.successRate;
|
|
1724
|
+
if (canary.appliedRuns >= 3 && (failureCanary > 0.050001 || failureCanary - failureBaseline >= 0.050001)) {
|
|
1725
|
+
triggers.push({
|
|
1726
|
+
metric: "failure-rate",
|
|
1727
|
+
baseline: failureBaseline,
|
|
1728
|
+
canary: failureCanary,
|
|
1729
|
+
threshold: failureCanary > 0.050001 ? ">5% absolute" : "+5pp regression"
|
|
1730
|
+
});
|
|
1731
|
+
}
|
|
1732
|
+
if (canary.avgQuality != null && (canary.avgQuality < 85 || baseline.avgQuality != null && baseline.avgQuality - canary.avgQuality >= 5)) {
|
|
1733
|
+
triggers.push({
|
|
1734
|
+
metric: "quality",
|
|
1735
|
+
baseline: baseline.avgQuality ?? 0,
|
|
1736
|
+
canary: canary.avgQuality,
|
|
1737
|
+
threshold: canary.avgQuality < 85 ? "<85 absolute" : "-5 points"
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
if (baseline.p95LatencyMs >= 1000 && canary.p95LatencyMs >= baseline.p95LatencyMs * 1.5) {
|
|
1741
|
+
triggers.push({ metric: "latency", baseline: baseline.p95LatencyMs, canary: canary.p95LatencyMs, threshold: "+50% p95" });
|
|
1742
|
+
}
|
|
1743
|
+
if (baseline.avgTokens >= 1000 && canary.avgTokens >= baseline.avgTokens * 1.5) {
|
|
1744
|
+
triggers.push({ metric: "tokens", baseline: baseline.avgTokens, canary: canary.avgTokens, threshold: "+50%" });
|
|
1745
|
+
}
|
|
1746
|
+
if (canary.fallbackRate - baseline.fallbackRate >= 0.1) {
|
|
1747
|
+
triggers.push({ metric: "fallback", baseline: baseline.fallbackRate, canary: canary.fallbackRate, threshold: "+10pp" });
|
|
1748
|
+
}
|
|
1749
|
+
if (canary.damageRate - baseline.damageRate >= 0.1) {
|
|
1750
|
+
triggers.push({ metric: "damage", baseline: baseline.damageRate, canary: canary.damageRate, threshold: "+10pp" });
|
|
1751
|
+
}
|
|
1752
|
+
const canarySampleAdequacy = Math.min(1, canary.appliedRuns / minCanaryRuns);
|
|
1753
|
+
const baselineSampleAdequacy = Math.min(1, baseline.appliedRuns / Math.max(20, minCanaryRuns));
|
|
1754
|
+
const dataConfidence = Math.round(100 * (canarySampleAdequacy * 0.25 + baselineSampleAdequacy * 0.15 + canary.qualityCoverage * canarySampleAdequacy * 0.2 + canary.damageCoverage * canarySampleAdequacy * 0.2 + baseline.damageCoverage * baselineSampleAdequacy * 0.2));
|
|
1755
|
+
const reasons = [];
|
|
1756
|
+
let decision = "hold";
|
|
1757
|
+
if (triggers.length && canary.appliedRuns >= 3) {
|
|
1758
|
+
decision = "rollback";
|
|
1759
|
+
reasons.push(...triggers.map((trigger) => trigger.metric + " crossed " + trigger.threshold));
|
|
1760
|
+
} else if (canary.appliedRuns < minCanaryRuns) {
|
|
1761
|
+
reasons.push("need " + (minCanaryRuns - canary.appliedRuns) + " more canary runs with applied outcomes");
|
|
1762
|
+
} else if (baseline.appliedRuns < Math.max(20, minCanaryRuns)) {
|
|
1763
|
+
reasons.push("stable baseline is too small");
|
|
1764
|
+
} else if (canary.qualityCoverage < 0.7) {
|
|
1765
|
+
reasons.push("schema-v2 quality coverage is below 70%");
|
|
1766
|
+
} else if (canary.damageCoverage < 0.7) {
|
|
1767
|
+
reasons.push("correlated canary damage-observation coverage is below 70%");
|
|
1768
|
+
} else if (baseline.damageCoverage < 0.7) {
|
|
1769
|
+
reasons.push("correlated stable damage-observation coverage is below 70%");
|
|
1770
|
+
} else if ((canary.avgQuality ?? 0) < 85) {
|
|
1771
|
+
reasons.push("absolute verifier quality is below 85");
|
|
1772
|
+
} else if (canary.successRate < 0.949999) {
|
|
1773
|
+
reasons.push("absolute success rate is below 95%");
|
|
1774
|
+
} else {
|
|
1775
|
+
decision = "promote";
|
|
1776
|
+
reasons.push("sample, absolute quality, reliability, latency, token, fallback, and damage gates passed");
|
|
1777
|
+
}
|
|
1778
|
+
return {
|
|
1779
|
+
version: options.version,
|
|
1780
|
+
decision,
|
|
1781
|
+
dataConfidence,
|
|
1782
|
+
baseline: roundStats(baseline),
|
|
1783
|
+
canary: roundStats(canary),
|
|
1784
|
+
triggers,
|
|
1785
|
+
reasons
|
|
1786
|
+
};
|
|
1787
|
+
}
|
|
1788
|
+
function safeMetricLabel(value, fallback) {
|
|
1789
|
+
if (typeof value !== "string" || !/^[\w./:@+-]{1,160}$/.test(value))
|
|
1790
|
+
return fallback;
|
|
1791
|
+
return value;
|
|
1792
|
+
}
|
|
1793
|
+
var TELEMETRY_FAILURE_KINDS = new Set([
|
|
1794
|
+
"cancelled",
|
|
1795
|
+
"timeout",
|
|
1796
|
+
"rate-limit",
|
|
1797
|
+
"authentication",
|
|
1798
|
+
"budget",
|
|
1799
|
+
"output-limit",
|
|
1800
|
+
"provider",
|
|
1801
|
+
"persistence",
|
|
1802
|
+
"validation",
|
|
1803
|
+
"verification",
|
|
1804
|
+
"yield",
|
|
1805
|
+
"internal"
|
|
1806
|
+
]);
|
|
1807
|
+
function isTelemetryFailureKind(value) {
|
|
1808
|
+
return typeof value === "string" && TELEMETRY_FAILURE_KINDS.has(value);
|
|
1809
|
+
}
|
|
1810
|
+
function buildPrivacySafeTelemetry(entries, damageEntries, options) {
|
|
1811
|
+
const groups = new Map;
|
|
1812
|
+
const failures = {};
|
|
1813
|
+
for (const entry of entries) {
|
|
1814
|
+
const version = safeMetricLabel(entry.version, "legacy");
|
|
1815
|
+
const channel = entry.releaseChannel === "canary" ? "canary" : "stable";
|
|
1816
|
+
const provider = safeMetricLabel(entry.provider, "unknown");
|
|
1817
|
+
const rawModel = safeMetricLabel(entry.model, "unknown");
|
|
1818
|
+
const model = rawModel.startsWith(provider + "/") ? rawModel.slice(provider.length + 1) : rawModel;
|
|
1819
|
+
const key = [version, channel, provider, model].join("\x00");
|
|
1820
|
+
const group = groups.get(key) ?? {
|
|
1821
|
+
version,
|
|
1822
|
+
channel,
|
|
1823
|
+
provider,
|
|
1824
|
+
model,
|
|
1825
|
+
runs: 0,
|
|
1826
|
+
successes: 0,
|
|
1827
|
+
quality: 0,
|
|
1828
|
+
qualityRuns: 0,
|
|
1829
|
+
latency: 0,
|
|
1830
|
+
input: 0,
|
|
1831
|
+
output: 0
|
|
1832
|
+
};
|
|
1833
|
+
group.runs++;
|
|
1834
|
+
if (entry.status === "success" || entry.status === "dry-run")
|
|
1835
|
+
group.successes++;
|
|
1836
|
+
if (entry.metricsSchemaVersion === 2 && typeof entry.verificationScore === "number") {
|
|
1837
|
+
group.quality += entry.verificationScore;
|
|
1838
|
+
group.qualityRuns++;
|
|
1839
|
+
}
|
|
1840
|
+
group.latency += entry.avgLatency;
|
|
1841
|
+
group.input += entry.totalInput + entry.totalCacheHit + (entry.totalCacheWrite ?? 0);
|
|
1842
|
+
group.output += entry.totalOutput;
|
|
1843
|
+
groups.set(key, group);
|
|
1844
|
+
if (isTelemetryFailureKind(entry.failureKind)) {
|
|
1845
|
+
failures[entry.failureKind] = (failures[entry.failureKind] ?? 0) + 1;
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
const aggregates = [...groups.values()].map((group) => ({
|
|
1849
|
+
version: group.version,
|
|
1850
|
+
channel: group.channel,
|
|
1851
|
+
provider: group.provider,
|
|
1852
|
+
model: group.model,
|
|
1853
|
+
runs: group.runs,
|
|
1854
|
+
successes: group.successes,
|
|
1855
|
+
avgQuality: group.qualityRuns ? Math.round(group.quality / group.qualityRuns * 10) / 10 : null,
|
|
1856
|
+
avgLatencyMs: group.runs ? Math.round(group.latency / group.runs) : 0,
|
|
1857
|
+
inputTokens: group.input,
|
|
1858
|
+
outputTokens: group.output
|
|
1859
|
+
})).sort((a, b) => b.runs - a.runs || a.version.localeCompare(b.version));
|
|
1860
|
+
return {
|
|
1861
|
+
generatedAt: new Date().toISOString(),
|
|
1862
|
+
totalRuns: entries.length,
|
|
1863
|
+
aggregates,
|
|
1864
|
+
failures,
|
|
1865
|
+
canary: assessCanary(entries, damageEntries, options),
|
|
1866
|
+
privacy: "aggregate-only; no session ids, project ids, prompts, summaries, paths, or error text"
|
|
1867
|
+
};
|
|
1868
|
+
}
|
|
1869
|
+
function formatPrivacySafeTelemetry(report) {
|
|
1870
|
+
const lines = [
|
|
1871
|
+
"# Smart Compact Telemetry",
|
|
1872
|
+
"",
|
|
1873
|
+
"Privacy: " + report.privacy + ".",
|
|
1874
|
+
"",
|
|
1875
|
+
"| Version | Channel | Provider/model | Runs | Success | Quality | Latency | Input | Output |",
|
|
1876
|
+
"|---|---|---|---:|---:|---:|---:|---:|---:|"
|
|
1877
|
+
];
|
|
1878
|
+
for (const item of report.aggregates) {
|
|
1879
|
+
lines.push("| " + item.version + " | " + item.channel + " | " + item.provider + "/" + item.model + " | " + item.runs + " | " + item.successes + "/" + item.runs + " | " + (item.avgQuality == null ? "n/a" : item.avgQuality.toFixed(1)) + " | " + item.avgLatencyMs + "ms | " + item.inputTokens + " | " + item.outputTokens + " |");
|
|
1880
|
+
}
|
|
1881
|
+
lines.push("", "## Canary: " + report.canary.decision.toUpperCase() + " (data confidence " + report.canary.dataConfidence + "%)", "");
|
|
1882
|
+
const baseline = report.canary.baseline;
|
|
1883
|
+
const canary = report.canary.canary;
|
|
1884
|
+
lines.push("| Gate | Stable baseline | Canary |", "|---|---:|---:|", "| Runs (total/applied) | " + baseline.runs + "/" + baseline.appliedRuns + " | " + canary.runs + "/" + canary.appliedRuns + " |", "| Success | " + Math.round(baseline.successRate * 100) + "% | " + Math.round(canary.successRate * 100) + "% |", "| Verify quality | " + (baseline.avgQuality ?? "n/a") + " | " + (canary.avgQuality ?? "n/a") + " |", "| p95 duration | " + baseline.p95LatencyMs + "ms | " + canary.p95LatencyMs + "ms |", "| Avg tokens | " + baseline.avgTokens + " | " + canary.avgTokens + " |", "| Fallback | " + Math.round(baseline.fallbackRate * 100) + "% | " + Math.round(canary.fallbackRate * 100) + "% |", "| Damage | " + Math.round(baseline.damageRate * 100) + "% | " + Math.round(canary.damageRate * 100) + "% |", "| Damage observed | " + Math.round(baseline.damageCoverage * 100) + "% | " + Math.round(canary.damageCoverage * 100) + "% |", "");
|
|
1885
|
+
for (const reason of report.canary.reasons)
|
|
1886
|
+
lines.push("- " + reason);
|
|
1887
|
+
const failureText = Object.entries(report.failures).map(([kind, count]) => kind + "=" + count).join(", ");
|
|
1888
|
+
lines.push("", "Failures: " + (failureText || "none classified"));
|
|
1889
|
+
return lines.join(`
|
|
1890
|
+
`);
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1567
1893
|
// src/utils/cache.ts
|
|
1568
1894
|
var INTERNAL_PHASES = new Set([
|
|
1569
1895
|
"explore-retry",
|
|
@@ -1693,8 +2019,9 @@ function hasListedPath(listed, file, display, normalizedOwners) {
|
|
|
1693
2019
|
}
|
|
1694
2020
|
return false;
|
|
1695
2021
|
}
|
|
1696
|
-
function outcomeClaims(summary) {
|
|
1697
|
-
|
|
2022
|
+
function outcomeClaims(summary, pathEvidence) {
|
|
2023
|
+
const pathLines = new Set(Array.from(pathEvidence, ([path2, display]) => [path2, display, "`" + path2 + "`"]).flat());
|
|
2024
|
+
return Array.from(new Set(summary.split(/\r?\n/).map((line) => line.replace(/^\s*(?:[-*+]|\d+[.)])\s+/, "").replace(/^\[[ x]\]\s+/i, "").trim()).filter((line) => line.length > 0 && !line.startsWith("#") && !pathLines.has(line)).filter((line) => HIGH_RISK_OUTCOME_RE.test(line)).filter((line) => /\bno\s+(?:errors?|failures?)\b/i.test(line) || !NEGATED_OUTCOME_RE.test(line)))).slice(0, 12);
|
|
1698
2025
|
}
|
|
1699
2026
|
function classifyOutcomeClaim(claim) {
|
|
1700
2027
|
const lower = claim.toLowerCase();
|
|
@@ -1928,7 +2255,7 @@ function stemToken(token) {
|
|
|
1928
2255
|
return lower;
|
|
1929
2256
|
}
|
|
1930
2257
|
function semanticTokens(text) {
|
|
1931
|
-
return (text.normalize("NFKC").match(/[\p{L}\p{N}_-]+/gu) ?? []).map(stemToken).filter((token) => token.length > 2);
|
|
2258
|
+
return (text.normalize("NFKC").match(/[\p{L}\p{N}_-]+/gu) ?? []).map(stemToken).filter((token) => token.length > 2 || NEGATION_MARKERS.has(token));
|
|
1932
2259
|
}
|
|
1933
2260
|
var semanticShapeCache = new Map;
|
|
1934
2261
|
var semanticFragmentCache = new Map;
|
|
@@ -1948,7 +2275,7 @@ function hasEffectiveTargetNegation(tokens, anchor) {
|
|
|
1948
2275
|
if (token !== anchor)
|
|
1949
2276
|
return false;
|
|
1950
2277
|
const nearbyStart = Math.max(0, anchorIndex - 2);
|
|
1951
|
-
const nearbyNegations = tokens.slice(nearbyStart, anchorIndex + 3).map((near, offset) => NEGATION_MARKERS.has(near) ? nearbyStart + offset : -1).filter((index) => index >= 0);
|
|
2278
|
+
const nearbyNegations = tokens.slice(nearbyStart, anchorIndex + 3).map((near, offset) => NEGATION_MARKERS.has(near) && !(near === "without" && nearbyStart + offset > anchorIndex) ? nearbyStart + offset : -1).filter((index) => index >= 0);
|
|
1952
2279
|
const governingStart = Math.max(0, anchorIndex - 3);
|
|
1953
2280
|
const preceding = tokens.slice(governingStart, anchorIndex);
|
|
1954
2281
|
const nearbyGuards = preceding.map((near, offset) => POLARITY_INVERTING_GUARDS.has(near) ? governingStart + offset : -1).filter((index) => index >= 0);
|
|
@@ -1997,12 +2324,13 @@ function hasSemanticEvidence(source, target) {
|
|
|
1997
2324
|
});
|
|
1998
2325
|
}
|
|
1999
2326
|
function hasSemanticContradiction(source, target) {
|
|
2327
|
+
const sourceFragments = new Set(semanticFragments(source).map((tokens) => tokens.join(" ")));
|
|
2000
2328
|
const { sourceTokens, concepts, anchor, negative, conditional } = semanticShape(source);
|
|
2001
2329
|
if (!anchor)
|
|
2002
2330
|
return false;
|
|
2003
2331
|
const required = Math.min(concepts.length, Math.max(1, Math.ceil(concepts.length * 0.6)));
|
|
2004
2332
|
return semanticFragments(target).some((tokens) => {
|
|
2005
|
-
if (!tokens.includes(anchor))
|
|
2333
|
+
if (!tokens.includes(anchor) || sourceFragments.has(tokens.join(" ")))
|
|
2006
2334
|
return false;
|
|
2007
2335
|
const overlap = concepts.filter((concept) => tokens.includes(concept)).length;
|
|
2008
2336
|
if (overlap < required)
|
|
@@ -2220,8 +2548,9 @@ function verifyFileReferences(summary, extraction, continuity, evidence, collect
|
|
|
2220
2548
|
...(continuity?.unresolvedErrors ?? []).flatMap((error) => error.files),
|
|
2221
2549
|
...(continuity?.openLoops ?? []).flatMap((loop) => loop.files)
|
|
2222
2550
|
]));
|
|
2551
|
+
const knownFileIndex = buildKnownPathReferenceIndex(knownFiles);
|
|
2223
2552
|
for (const ref of new Set(extractFileRefs(summary))) {
|
|
2224
|
-
const grounded =
|
|
2553
|
+
const grounded = isKnownPathReferenceInIndex(ref, knownFileIndex) || Boolean(evidence.sourceMessages && sourceSupportsFileReference(ref, evidence.sourceMessages));
|
|
2225
2554
|
if (!grounded)
|
|
2226
2555
|
addGap(accumulator, { kind: "fabricated-file", ref }, 4);
|
|
2227
2556
|
}
|
|
@@ -2239,9 +2568,10 @@ function verifyProgressConsistency(parsed, extraction, collected, paths, accumul
|
|
|
2239
2568
|
}, 12);
|
|
2240
2569
|
}
|
|
2241
2570
|
const doneRefs = new Set(extractFileRefs(done).map(normalizePath));
|
|
2571
|
+
const modifiedPathOwners = buildPathNeedleOwnershipIndex(paths.modified);
|
|
2242
2572
|
for (const file of extraction.modifiedFiles) {
|
|
2243
|
-
const needles =
|
|
2244
|
-
if (!needles.some((needle) => doneRefs.has(
|
|
2573
|
+
const needles = buildUniquePathNeedlesFromIndex(file.path, modifiedPathOwners);
|
|
2574
|
+
if (!needles.some((needle) => doneRefs.has(needle)))
|
|
2245
2575
|
continue;
|
|
2246
2576
|
const unresolved = collected.unresolved.find((error) => {
|
|
2247
2577
|
const firstLine = error.message.split(/\r?\n/, 1)[0] ?? "";
|
|
@@ -2256,7 +2586,7 @@ function verifyProgressConsistency(parsed, extraction, collected, paths, accumul
|
|
|
2256
2586
|
}
|
|
2257
2587
|
}
|
|
2258
2588
|
}
|
|
2259
|
-
function verifyOpenLoopsAndClaims(summary, parsed, extraction, continuity, evidence, collected, accumulator) {
|
|
2589
|
+
function verifyOpenLoopsAndClaims(summary, parsed, paths, extraction, continuity, evidence, collected, accumulator) {
|
|
2260
2590
|
const unresolvedCount = collected.unresolved.length + (continuity?.openLoops.filter((loop) => loop.status !== "resolved").length ?? 0);
|
|
2261
2591
|
if (unresolvedCount >= 1 && !findSection(parsed, "open-loops") && !summary.toLowerCase().replace(/\\/g, "/").includes("unresolved")) {
|
|
2262
2592
|
addGap(accumulator, { kind: "missing-open-loops", unresolvedCount }, 5);
|
|
@@ -2264,7 +2594,7 @@ function verifyOpenLoopsAndClaims(summary, parsed, extraction, continuity, evide
|
|
|
2264
2594
|
if (!evidence.sourceMessages)
|
|
2265
2595
|
return;
|
|
2266
2596
|
const tools = successfulToolEvidence(evidence.sourceMessages);
|
|
2267
|
-
for (const claim of outcomeClaims(summary)) {
|
|
2597
|
+
for (const claim of outcomeClaims(summary, paths.rendered)) {
|
|
2268
2598
|
if (!successfulToolSupportsClaim(claim, tools, extraction)) {
|
|
2269
2599
|
addGap(accumulator, { kind: "unsupported-claim", claim }, 20);
|
|
2270
2600
|
}
|
|
@@ -2280,7 +2610,7 @@ function verifySummary(summary, extraction, continuity = null, evidence = {}) {
|
|
|
2280
2610
|
verifySemanticCoverage(parsed, collected, accumulator);
|
|
2281
2611
|
verifyFileReferences(summary, extraction, continuity, evidence, collected, paths, accumulator);
|
|
2282
2612
|
verifyProgressConsistency(parsed, extraction, collected, paths, accumulator);
|
|
2283
|
-
verifyOpenLoopsAndClaims(summary, parsed, extraction, continuity, evidence, collected, accumulator);
|
|
2613
|
+
verifyOpenLoopsAndClaims(summary, parsed, paths, extraction, continuity, evidence, collected, accumulator);
|
|
2284
2614
|
const score = Math.max(0, accumulator.score);
|
|
2285
2615
|
return {
|
|
2286
2616
|
ok: accumulator.gaps.length === 0 && score >= 85,
|