code-context-control 2.28.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/__init__.py +1 -0
- cli/_hook_utils.py +99 -0
- cli/c3.py +6152 -0
- cli/commands/__init__.py +1 -0
- cli/commands/common.py +312 -0
- cli/commands/parser.py +286 -0
- cli/docs.html +3178 -0
- cli/edits.html +878 -0
- cli/hook_auto_snapshot.py +142 -0
- cli/hook_c3_signal.py +61 -0
- cli/hook_c3read.py +116 -0
- cli/hook_edit_ledger.py +213 -0
- cli/hook_edit_unlock.py +170 -0
- cli/hook_filter.py +130 -0
- cli/hook_ghost_files.py +238 -0
- cli/hook_pretool_enforce.py +334 -0
- cli/hook_read.py +200 -0
- cli/hook_session_stats.py +62 -0
- cli/hook_terse_advisor.py +190 -0
- cli/hub.html +3764 -0
- cli/hub_server.py +1619 -0
- cli/mcp_proxy.py +428 -0
- cli/mcp_server.py +660 -0
- cli/server.py +2985 -0
- cli/tools/__init__.py +4 -0
- cli/tools/_helpers.py +65 -0
- cli/tools/agent.py +1165 -0
- cli/tools/compress.py +215 -0
- cli/tools/delegate.py +1184 -0
- cli/tools/edit.py +313 -0
- cli/tools/edits.py +118 -0
- cli/tools/filter.py +285 -0
- cli/tools/impact.py +163 -0
- cli/tools/memory.py +469 -0
- cli/tools/read.py +224 -0
- cli/tools/search.py +337 -0
- cli/tools/session.py +95 -0
- cli/tools/shell.py +193 -0
- cli/tools/status.py +306 -0
- cli/tools/validate.py +310 -0
- cli/ui/api.js +36 -0
- cli/ui/app.js +207 -0
- cli/ui/components/chat.js +758 -0
- cli/ui/components/dashboard.js +689 -0
- cli/ui/components/edits.js +220 -0
- cli/ui/components/instructions.js +481 -0
- cli/ui/components/memory.js +626 -0
- cli/ui/components/sessions.js +606 -0
- cli/ui/components/settings.js +1404 -0
- cli/ui/components/sidebar.js +156 -0
- cli/ui/icons.js +51 -0
- cli/ui/shared.js +119 -0
- cli/ui/theme.js +22 -0
- cli/ui.html +168 -0
- cli/ui_legacy.html +6797 -0
- cli/ui_nano.html +503 -0
- code_context_control-2.28.0.dist-info/METADATA +248 -0
- code_context_control-2.28.0.dist-info/RECORD +150 -0
- code_context_control-2.28.0.dist-info/WHEEL +5 -0
- code_context_control-2.28.0.dist-info/entry_points.txt +4 -0
- code_context_control-2.28.0.dist-info/licenses/LICENSE +201 -0
- code_context_control-2.28.0.dist-info/top_level.txt +5 -0
- core/__init__.py +75 -0
- core/config.py +269 -0
- core/ide.py +188 -0
- oracle/__init__.py +1 -0
- oracle/config.py +75 -0
- oracle/oracle.html +3900 -0
- oracle/oracle_server.py +663 -0
- oracle/services/__init__.py +1 -0
- oracle/services/c3_bridge.py +210 -0
- oracle/services/chat_engine.py +1103 -0
- oracle/services/chat_store.py +155 -0
- oracle/services/cross_memory.py +154 -0
- oracle/services/federated_graph.py +463 -0
- oracle/services/health_checker.py +117 -0
- oracle/services/insight_engine.py +307 -0
- oracle/services/memory_reader.py +106 -0
- oracle/services/memory_writer.py +182 -0
- oracle/services/ollama_bridge.py +332 -0
- oracle/services/project_scanner.py +87 -0
- oracle/services/review_agent.py +206 -0
- services/__init__.py +1 -0
- services/activity_log.py +93 -0
- services/agent_base.py +124 -0
- services/agents.py +1529 -0
- services/auto_memory.py +407 -0
- services/bench/__init__.py +6 -0
- services/bench/external/__init__.py +29 -0
- services/bench/external/aider_polyglot.py +405 -0
- services/bench/external/swe_bench.py +485 -0
- services/benchmark_dashboard.py +596 -0
- services/claude_md.py +785 -0
- services/compressor.py +592 -0
- services/context_snapshot.py +356 -0
- services/conversation_store.py +870 -0
- services/doc_index.py +537 -0
- services/e2e_benchmark.py +2884 -0
- services/e2e_evaluator.py +396 -0
- services/e2e_tasks.py +743 -0
- services/edit_ledger.py +459 -0
- services/embedding_index.py +341 -0
- services/error_reporting.py +123 -0
- services/file_memory.py +734 -0
- services/hub_service.py +585 -0
- services/indexer.py +712 -0
- services/memory.py +318 -0
- services/memory_consolidator.py +538 -0
- services/memory_graph.py +382 -0
- services/memory_grounder.py +304 -0
- services/memory_scorer.py +246 -0
- services/metrics.py +86 -0
- services/notifications.py +209 -0
- services/ollama_client.py +201 -0
- services/output_filter.py +488 -0
- services/parser.py +1238 -0
- services/project_manager.py +579 -0
- services/protocol.py +306 -0
- services/proxy_state.py +152 -0
- services/retrieval_broker.py +129 -0
- services/router.py +414 -0
- services/runtime.py +326 -0
- services/session_benchmark.py +1945 -0
- services/session_manager.py +1026 -0
- services/session_preloader.py +251 -0
- services/text_index.py +90 -0
- services/tool_classifier.py +176 -0
- services/transcript_index.py +340 -0
- services/validation_cache.py +155 -0
- services/vector_store.py +299 -0
- services/version_tracker.py +271 -0
- services/watcher.py +192 -0
- tui/__init__.py +0 -0
- tui/backend.py +59 -0
- tui/main.py +145 -0
- tui/screens/__init__.py +1 -0
- tui/screens/benchmark_view.py +109 -0
- tui/screens/claudemd_view.py +46 -0
- tui/screens/compress_view.py +52 -0
- tui/screens/index_view.py +74 -0
- tui/screens/init_view.py +82 -0
- tui/screens/mcp_view.py +73 -0
- tui/screens/optimize_view.py +41 -0
- tui/screens/pipe_view.py +46 -0
- tui/screens/projects_view.py +355 -0
- tui/screens/search_view.py +55 -0
- tui/screens/session_view.py +143 -0
- tui/screens/stats.py +158 -0
- tui/screens/ui_view.py +54 -0
- tui/theme.tcss +335 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Thin HTTP client for Ollama — stdlib only, no new dependencies.
|
|
2
|
+
|
|
3
|
+
All calls have a 10s timeout and return None on failure,
|
|
4
|
+
so callers can gracefully degrade to non-LLM paths.
|
|
5
|
+
|
|
6
|
+
Includes integrated LLM response cache (formerly llm_cache.py).
|
|
7
|
+
"""
|
|
8
|
+
import hashlib
|
|
9
|
+
import json
|
|
10
|
+
import urllib.error
|
|
11
|
+
import urllib.request
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Optional
|
|
14
|
+
|
|
15
|
+
DEFAULT_BASE_URL = "http://localhost:11434"
|
|
16
|
+
_TIMEOUT = 30 # seconds
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class LLMCache:
|
|
20
|
+
"""Persistent disk cache for LLM results."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, cache_dir: str = ".c3/cache/llm"):
|
|
23
|
+
self.cache_dir = Path(cache_dir)
|
|
24
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
25
|
+
|
|
26
|
+
def _get_key(self, prompt: str, model: str, system: str = "", **options) -> str:
|
|
27
|
+
opt_str = json.dumps(options, sort_keys=True)
|
|
28
|
+
key_content = f"{model}:{system}:{prompt}:{opt_str}"
|
|
29
|
+
return hashlib.md5(key_content.encode()).hexdigest()
|
|
30
|
+
|
|
31
|
+
def get(self, prompt: str, model: str, system: str = "", **options) -> Optional[str]:
|
|
32
|
+
key = self._get_key(prompt, model, system, **options)
|
|
33
|
+
cache_file = self.cache_dir / f"{key}.json"
|
|
34
|
+
if cache_file.exists():
|
|
35
|
+
try:
|
|
36
|
+
with open(cache_file, "r", encoding="utf-8") as f:
|
|
37
|
+
data = json.load(f)
|
|
38
|
+
return data.get("response")
|
|
39
|
+
except Exception:
|
|
40
|
+
pass
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
def set(self, prompt: str, model: str, response: str, system: str = "", **options):
|
|
44
|
+
key = self._get_key(prompt, model, system, **options)
|
|
45
|
+
cache_file = self.cache_dir / f"{key}.json"
|
|
46
|
+
try:
|
|
47
|
+
with open(cache_file, "w", encoding="utf-8") as f:
|
|
48
|
+
json.dump({
|
|
49
|
+
"model": model,
|
|
50
|
+
"system": system,
|
|
51
|
+
"prompt": prompt,
|
|
52
|
+
"options": options,
|
|
53
|
+
"response": response
|
|
54
|
+
}, f, indent=2)
|
|
55
|
+
except Exception:
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class OllamaClient:
|
|
60
|
+
"""Minimal Ollama REST client using urllib."""
|
|
61
|
+
|
|
62
|
+
_AVAIL_TTL_SECONDS = 10.0
|
|
63
|
+
|
|
64
|
+
def __init__(self, base_url: str = DEFAULT_BASE_URL, cache_dir: str = ".c3/cache/llm"):
|
|
65
|
+
self.base_url = base_url.rstrip("/")
|
|
66
|
+
self.cache = LLMCache(cache_dir)
|
|
67
|
+
self._avail_value: bool | None = None
|
|
68
|
+
self._avail_ts: float = 0.0
|
|
69
|
+
|
|
70
|
+
# ── Availability ──────────────────────────────────────
|
|
71
|
+
|
|
72
|
+
def is_available(self, timeout: int | None = None) -> bool:
|
|
73
|
+
"""Check if Ollama is reachable. Cached for ``_AVAIL_TTL_SECONDS`` to
|
|
74
|
+
avoid storm when many background agents poll concurrently."""
|
|
75
|
+
import time
|
|
76
|
+
now = time.monotonic()
|
|
77
|
+
if self._avail_value is not None and (now - self._avail_ts) < self._AVAIL_TTL_SECONDS:
|
|
78
|
+
return self._avail_value
|
|
79
|
+
try:
|
|
80
|
+
req = urllib.request.Request(f"{self.base_url}/api/tags")
|
|
81
|
+
with urllib.request.urlopen(req, timeout=timeout or _TIMEOUT):
|
|
82
|
+
self._avail_value = True
|
|
83
|
+
except Exception:
|
|
84
|
+
self._avail_value = False
|
|
85
|
+
self._avail_ts = now
|
|
86
|
+
return self._avail_value
|
|
87
|
+
|
|
88
|
+
# ── Models ────────────────────────────────────────────
|
|
89
|
+
|
|
90
|
+
def list_models(self) -> list[str] | None:
|
|
91
|
+
"""Return list of locally available model names, or None on failure."""
|
|
92
|
+
try:
|
|
93
|
+
req = urllib.request.Request(f"{self.base_url}/api/tags")
|
|
94
|
+
with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp:
|
|
95
|
+
data = json.loads(resp.read())
|
|
96
|
+
return [m["name"] for m in data.get("models", [])]
|
|
97
|
+
except Exception:
|
|
98
|
+
return None
|
|
99
|
+
|
|
100
|
+
def has_model(self, model: str) -> bool:
|
|
101
|
+
"""Check if a specific model is pulled locally."""
|
|
102
|
+
models = self.list_models()
|
|
103
|
+
if models is None:
|
|
104
|
+
return False
|
|
105
|
+
return any(model in m or m.startswith(model) for m in models)
|
|
106
|
+
|
|
107
|
+
# ── Embeddings ────────────────────────────────────────
|
|
108
|
+
|
|
109
|
+
def embed(self, text: str, model: str = "nomic-embed-text") -> list[float] | None:
|
|
110
|
+
"""Generate embedding vector for text. Returns None on failure."""
|
|
111
|
+
try:
|
|
112
|
+
payload = json.dumps({"model": model, "input": text}).encode()
|
|
113
|
+
req = urllib.request.Request(
|
|
114
|
+
f"{self.base_url}/api/embed",
|
|
115
|
+
data=payload,
|
|
116
|
+
headers={"Content-Type": "application/json"},
|
|
117
|
+
method="POST",
|
|
118
|
+
)
|
|
119
|
+
with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp:
|
|
120
|
+
data = json.loads(resp.read())
|
|
121
|
+
embeddings = data.get("embeddings")
|
|
122
|
+
if embeddings and len(embeddings) > 0:
|
|
123
|
+
return embeddings[0]
|
|
124
|
+
return None
|
|
125
|
+
except Exception:
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
def embed_batch(self, texts: list[str], model: str = "nomic-embed-text") -> list[list[float]] | None:
|
|
129
|
+
"""Embed multiple texts in one call. Returns None on failure."""
|
|
130
|
+
try:
|
|
131
|
+
payload = json.dumps({"model": model, "input": texts}).encode()
|
|
132
|
+
req = urllib.request.Request(
|
|
133
|
+
f"{self.base_url}/api/embed",
|
|
134
|
+
data=payload,
|
|
135
|
+
headers={"Content-Type": "application/json"},
|
|
136
|
+
method="POST",
|
|
137
|
+
)
|
|
138
|
+
with urllib.request.urlopen(req, timeout=_TIMEOUT * 3) as resp:
|
|
139
|
+
data = json.loads(resp.read())
|
|
140
|
+
return data.get("embeddings")
|
|
141
|
+
except Exception:
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
# ── Generation ────────────────────────────────────────
|
|
145
|
+
|
|
146
|
+
def generate(self, prompt: str, model: str = "gemma3n:latest",
|
|
147
|
+
system: str = "", temperature: float = 0.3,
|
|
148
|
+
max_tokens: int = 512, num_ctx: int = 4096,
|
|
149
|
+
stream: bool = False, timeout: int = 60):
|
|
150
|
+
"""Generate text completion. Returns string if stream=False, or generator if True."""
|
|
151
|
+
options = {"temperature": temperature, "num_predict": max_tokens, "num_ctx": num_ctx}
|
|
152
|
+
|
|
153
|
+
if not stream:
|
|
154
|
+
cached = self.cache.get(prompt, model, system, **options)
|
|
155
|
+
if cached:
|
|
156
|
+
return cached
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
body = {
|
|
160
|
+
"model": model,
|
|
161
|
+
"prompt": prompt,
|
|
162
|
+
"stream": stream,
|
|
163
|
+
"options": options,
|
|
164
|
+
}
|
|
165
|
+
if system:
|
|
166
|
+
body["system"] = system
|
|
167
|
+
payload = json.dumps(body).encode()
|
|
168
|
+
req = urllib.request.Request(
|
|
169
|
+
f"{self.base_url}/api/generate",
|
|
170
|
+
data=payload,
|
|
171
|
+
headers={"Content-Type": "application/json"},
|
|
172
|
+
method="POST",
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
if stream:
|
|
176
|
+
return self._stream_generator(req)
|
|
177
|
+
|
|
178
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
179
|
+
data = json.loads(resp.read())
|
|
180
|
+
|
|
181
|
+
response = data.get("response")
|
|
182
|
+
if response:
|
|
183
|
+
self.cache.set(prompt, model, response, system, **options)
|
|
184
|
+
return response
|
|
185
|
+
except Exception:
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
def _stream_generator(self, request):
|
|
189
|
+
"""Internal generator for streaming responses."""
|
|
190
|
+
try:
|
|
191
|
+
with urllib.request.urlopen(request, timeout=60) as resp:
|
|
192
|
+
for line in resp:
|
|
193
|
+
if not line:
|
|
194
|
+
continue
|
|
195
|
+
chunk = json.loads(line.decode("utf-8"))
|
|
196
|
+
if "response" in chunk:
|
|
197
|
+
yield chunk["response"]
|
|
198
|
+
if chunk.get("done"):
|
|
199
|
+
break
|
|
200
|
+
except Exception as e:
|
|
201
|
+
yield f"\n[Streaming Error: {e}]"
|
|
@@ -0,0 +1,488 @@
|
|
|
1
|
+
"""Terminal Output Filter — Two-pass pipeline for reducing terminal noise.
|
|
2
|
+
|
|
3
|
+
Pass 1 (always): Strip ANSI, collapse progress bars, deduplicate PASS/OK lines,
|
|
4
|
+
collapse repeated lines, normalize blanks.
|
|
5
|
+
Pass 2 (optional): If pass1 output > threshold tokens, use Ollama LLM for
|
|
6
|
+
3-5 line summary. Status-aware: success=terse, failure=preserve errors.
|
|
7
|
+
"""
|
|
8
|
+
import re
|
|
9
|
+
import threading
|
|
10
|
+
from collections import Counter, defaultdict
|
|
11
|
+
|
|
12
|
+
from core import count_tokens
|
|
13
|
+
from services.ollama_client import OllamaClient
|
|
14
|
+
|
|
15
|
+
# ── ANSI escape regex ────────────────────────────────────
|
|
16
|
+
_ANSI_RE = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]|\x1b\].*?\x07|\x1b\(B')
|
|
17
|
+
|
|
18
|
+
# ── Progress bar patterns ────────────────────────────────
|
|
19
|
+
_PROGRESS_RE = re.compile(
|
|
20
|
+
r'[\s]*[\|#=\-\>\.]{5,}[\s]*\d+%' # ||||||||| 50%
|
|
21
|
+
r'|[\s]*\d+%[\s]*[\|#=\-\>\.]{5,}' # 50% |||||||||
|
|
22
|
+
r'|[\s]*\d+/\d+[\s]*[\|#=\-\>\.]{3,}' # 5/10 ||||
|
|
23
|
+
r'|[\s]*[\u2588\u2591\u2592\u2593]{3,}' # Unicode blocks
|
|
24
|
+
r'|\r[^\n]*\d+%' # Carriage-return progress
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# ── Pass/OK line patterns ────────────────────────────────
|
|
28
|
+
_PASS_RE = re.compile(
|
|
29
|
+
r'^\s*(PASS|PASSED|OK|ok|\u2713|✓|\.)\s+'
|
|
30
|
+
r'|^\s*test[_\s].*\.\.\.\s*(ok|PASS)',
|
|
31
|
+
re.IGNORECASE,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# ── Error/failure patterns (to preserve) ─────────────────
|
|
35
|
+
_ERROR_RE = re.compile(
|
|
36
|
+
r'ERROR|FAIL|FAILED|Exception|Traceback|panic|CRITICAL'
|
|
37
|
+
r'|error\[|warning\[|^\s*E\s+'
|
|
38
|
+
r'|assert|AssertionError|TypeError|ValueError|KeyError'
|
|
39
|
+
r'|ModuleNotFoundError|ImportError|FileNotFoundError',
|
|
40
|
+
re.IGNORECASE,
|
|
41
|
+
)
|
|
42
|
+
_PYTEST_PASS_RE = re.compile(
|
|
43
|
+
r'^(?P<target>[^\s:][^:]*(?:::[^\s:]+)+)\s+(?P<status>PASSED|PASS|OK)\b',
|
|
44
|
+
re.IGNORECASE,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# ── Package manager noise (npm, pip, cargo) ──────────────
|
|
48
|
+
_PKG_NOISE_RE = re.compile(
|
|
49
|
+
r'^(npm (http fetch|notice|verb|WARN)|'
|
|
50
|
+
r'Requirement already satisfied:|'
|
|
51
|
+
r'\s*(Downloading|Fetching|Compiling|Building|Updating|Installed)\s+'
|
|
52
|
+
r'|.*=> (Resolving|Downloading|Building|installing))',
|
|
53
|
+
re.IGNORECASE,
|
|
54
|
+
)
|
|
55
|
+
_SUCCESS_SUMMARY_RE = re.compile(
|
|
56
|
+
r'('
|
|
57
|
+
r'collected\s+\d+\s+items?'
|
|
58
|
+
r'|=+.*\b(passed|failed|warnings?)\b.*=+'
|
|
59
|
+
r'|\b\d+\s+passed\b'
|
|
60
|
+
r'|\b\d+\s+failed\b'
|
|
61
|
+
r'|\b\d+\s+warnings?\b'
|
|
62
|
+
r'|\bran\s+\d+\s+tests?\s+in\s+'
|
|
63
|
+
r'|\b(build|built|compile|compiled|bundle|bundled|finish|finished|done)\b'
|
|
64
|
+
r'|\bin\s+\d+(?:\.\d+)?s\b'
|
|
65
|
+
r'|\badded\s+\d+\s+packages?\b'
|
|
66
|
+
r'|\binstalled\b'
|
|
67
|
+
r')',
|
|
68
|
+
re.IGNORECASE,
|
|
69
|
+
)
|
|
70
|
+
_WARNING_RE = re.compile(r'\b(warning|warn)\b', re.IGNORECASE)
|
|
71
|
+
|
|
72
|
+
class OutputFilter:
|
|
73
|
+
"""Two-pass terminal output filter with optional LLM summarization."""
|
|
74
|
+
|
|
75
|
+
def __init__(self, config: dict | None = None):
|
|
76
|
+
self.config = config or {}
|
|
77
|
+
base_url = self.config.get("ollama_base_url", "http://localhost:11434")
|
|
78
|
+
self.ollama = OllamaClient(base_url)
|
|
79
|
+
self.filter_model = self.config.get("filter_model", "gemma3n:latest")
|
|
80
|
+
self.llm_threshold = self.config.get("filter_llm_threshold", 500)
|
|
81
|
+
self._lock = threading.Lock()
|
|
82
|
+
|
|
83
|
+
# Metrics
|
|
84
|
+
self.metrics = {
|
|
85
|
+
"calls": 0,
|
|
86
|
+
"raw_tokens": 0,
|
|
87
|
+
"filtered_tokens": 0,
|
|
88
|
+
"llm_calls": 0,
|
|
89
|
+
"total_savings_pct": 0.0,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
def filter(self, text: str, use_llm: bool = True) -> dict:
|
|
93
|
+
"""Run the two-pass filter pipeline.
|
|
94
|
+
|
|
95
|
+
Returns dict with: filtered, raw_tokens, filtered_tokens, savings_pct,
|
|
96
|
+
pass_used (1 or 2), llm_used (bool)
|
|
97
|
+
"""
|
|
98
|
+
if not text or not text.strip():
|
|
99
|
+
return {
|
|
100
|
+
"filtered": text,
|
|
101
|
+
"raw_tokens": 0,
|
|
102
|
+
"filtered_tokens": 0,
|
|
103
|
+
"savings_pct": 0,
|
|
104
|
+
"pass_used": 0,
|
|
105
|
+
"llm_used": False,
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
raw_tokens = count_tokens(text)
|
|
109
|
+
|
|
110
|
+
# Pass 1: deterministic filtering
|
|
111
|
+
pass1 = self._pass1(text)
|
|
112
|
+
pass1_tokens = count_tokens(pass1)
|
|
113
|
+
mode = self._detect_mode(pass1.splitlines())
|
|
114
|
+
|
|
115
|
+
result_text = pass1
|
|
116
|
+
llm_used = False
|
|
117
|
+
pass_used = 1
|
|
118
|
+
|
|
119
|
+
# Compact verbose output even when LLM summarization is disabled.
|
|
120
|
+
compact = self._summarize_signal_output(pass1, mode=mode)
|
|
121
|
+
if compact and count_tokens(compact) < pass1_tokens:
|
|
122
|
+
result_text = compact
|
|
123
|
+
pass1_tokens = count_tokens(compact)
|
|
124
|
+
|
|
125
|
+
# Pass 2: LLM summarization if still too large
|
|
126
|
+
if use_llm and pass1_tokens > self.llm_threshold:
|
|
127
|
+
if not self.config.get("HYBRID_DISABLE_TIER1"):
|
|
128
|
+
llm_result = self._pass2(result_text, raw_text=text)
|
|
129
|
+
if llm_result:
|
|
130
|
+
result_text = llm_result
|
|
131
|
+
llm_used = True
|
|
132
|
+
pass_used = 2
|
|
133
|
+
|
|
134
|
+
filtered_tokens = count_tokens(result_text)
|
|
135
|
+
savings_pct = round((1 - filtered_tokens / raw_tokens) * 100, 1) if raw_tokens > 0 else 0
|
|
136
|
+
|
|
137
|
+
# Update metrics
|
|
138
|
+
with self._lock:
|
|
139
|
+
self.metrics["calls"] += 1
|
|
140
|
+
self.metrics["raw_tokens"] += raw_tokens
|
|
141
|
+
self.metrics["filtered_tokens"] += filtered_tokens
|
|
142
|
+
if llm_used:
|
|
143
|
+
self.metrics["llm_calls"] += 1
|
|
144
|
+
total_raw = self.metrics["raw_tokens"]
|
|
145
|
+
total_filt = self.metrics["filtered_tokens"]
|
|
146
|
+
self.metrics["total_savings_pct"] = round(
|
|
147
|
+
(1 - total_filt / total_raw) * 100, 1
|
|
148
|
+
) if total_raw > 0 else 0
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
"filtered": result_text,
|
|
152
|
+
"raw_tokens": raw_tokens,
|
|
153
|
+
"filtered_tokens": filtered_tokens,
|
|
154
|
+
"savings_pct": savings_pct,
|
|
155
|
+
"pass_used": pass_used,
|
|
156
|
+
"llm_used": llm_used,
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
def get_metrics(self) -> dict:
|
|
160
|
+
"""Return accumulated filter metrics."""
|
|
161
|
+
with self._lock:
|
|
162
|
+
return dict(self.metrics)
|
|
163
|
+
|
|
164
|
+
# ── Pass 1: Deterministic filtering ──────────────────
|
|
165
|
+
|
|
166
|
+
def _pass1(self, text: str) -> str:
|
|
167
|
+
"""Strip ANSI, collapse progress, deduplicate PASS lines, collapse repeats."""
|
|
168
|
+
lines = text.splitlines()
|
|
169
|
+
mode = self._detect_mode(lines)
|
|
170
|
+
|
|
171
|
+
# Strip ANSI codes
|
|
172
|
+
lines = [_ANSI_RE.sub('', line) for line in lines]
|
|
173
|
+
|
|
174
|
+
# Remove pure progress bar lines and package manager noise
|
|
175
|
+
lines = [line for line in lines if not _PROGRESS_RE.fullmatch(line) and not _PKG_NOISE_RE.search(line)]
|
|
176
|
+
|
|
177
|
+
# Collapse tracebacks before other summarization passes.
|
|
178
|
+
lines = self._collapse_tracebacks(lines)
|
|
179
|
+
|
|
180
|
+
# Collapse PASS/OK lines, with stronger grouping for test output.
|
|
181
|
+
lines = self._collapse_pass_lines(lines, mode=mode)
|
|
182
|
+
|
|
183
|
+
# Collapse repeated identical lines, then noisy repeats across the full output.
|
|
184
|
+
lines = self._collapse_repeats(lines)
|
|
185
|
+
lines = self._collapse_global_repeats(lines, mode=mode)
|
|
186
|
+
|
|
187
|
+
# Normalize multiple blank lines to single
|
|
188
|
+
lines = self._normalize_blanks(lines)
|
|
189
|
+
|
|
190
|
+
return '\n'.join(lines)
|
|
191
|
+
|
|
192
|
+
def _detect_mode(self, lines: list[str]) -> str:
|
|
193
|
+
"""Infer a coarse output mode so filtering can be more aggressive."""
|
|
194
|
+
sample = "\n".join(lines[:120]).lower()
|
|
195
|
+
if "passed" in sample and ("pytest" in sample or "tests/" in sample or "::test_" in sample):
|
|
196
|
+
return "test"
|
|
197
|
+
if "downloading" in sample or "installing" in sample or "fetching" in sample:
|
|
198
|
+
return "install"
|
|
199
|
+
if "build" in sample or "compil" in sample or "bundl" in sample:
|
|
200
|
+
return "build"
|
|
201
|
+
return "generic"
|
|
202
|
+
|
|
203
|
+
def _collapse_pass_lines(self, lines: list[str], mode: str = "generic") -> list[str]:
|
|
204
|
+
"""Replace consecutive PASS/OK lines with a summary count."""
|
|
205
|
+
if mode == "test":
|
|
206
|
+
return self._collapse_test_pass_lines(lines)
|
|
207
|
+
|
|
208
|
+
result = []
|
|
209
|
+
pass_count = 0
|
|
210
|
+
pass_run_start = -1
|
|
211
|
+
|
|
212
|
+
for i, line in enumerate(lines):
|
|
213
|
+
if _PASS_RE.search(line) and not _ERROR_RE.search(line):
|
|
214
|
+
if pass_count == 0:
|
|
215
|
+
pass_run_start = i
|
|
216
|
+
pass_count += 1
|
|
217
|
+
else:
|
|
218
|
+
if pass_count > 3:
|
|
219
|
+
result.append(f"[{pass_count} tests passed]")
|
|
220
|
+
elif pass_count > 0:
|
|
221
|
+
# Keep small groups as-is
|
|
222
|
+
result.extend(lines[pass_run_start:pass_run_start + pass_count])
|
|
223
|
+
pass_count = 0
|
|
224
|
+
result.append(line)
|
|
225
|
+
|
|
226
|
+
# Handle trailing pass lines
|
|
227
|
+
if pass_count > 3:
|
|
228
|
+
result.append(f"[{pass_count} tests passed]")
|
|
229
|
+
elif pass_count > 0:
|
|
230
|
+
result.extend(lines[pass_run_start:pass_run_start + pass_count])
|
|
231
|
+
|
|
232
|
+
return result
|
|
233
|
+
|
|
234
|
+
def _collapse_test_pass_lines(self, lines: list[str]) -> list[str]:
|
|
235
|
+
"""Replace large pytest-style pass output with grouped summaries."""
|
|
236
|
+
result = []
|
|
237
|
+
pending_passes = []
|
|
238
|
+
|
|
239
|
+
def flush_pending():
|
|
240
|
+
nonlocal pending_passes
|
|
241
|
+
if not pending_passes:
|
|
242
|
+
return
|
|
243
|
+
if len(pending_passes) <= 3:
|
|
244
|
+
result.extend(pending_passes)
|
|
245
|
+
else:
|
|
246
|
+
grouped = defaultdict(int)
|
|
247
|
+
for line in pending_passes:
|
|
248
|
+
match = _PYTEST_PASS_RE.match(line.strip())
|
|
249
|
+
if match:
|
|
250
|
+
target = match.group("target")
|
|
251
|
+
key = target.split("::", 1)[0]
|
|
252
|
+
else:
|
|
253
|
+
key = "[misc]"
|
|
254
|
+
grouped[key] += 1
|
|
255
|
+
summary_parts = [f"{count} in {name}" for name, count in sorted(grouped.items())]
|
|
256
|
+
result.append(f"[{len(pending_passes)} tests passed] " + " | ".join(summary_parts[:6]))
|
|
257
|
+
if len(summary_parts) > 6:
|
|
258
|
+
result.append(f"[{len(summary_parts) - 6} more test groups omitted]")
|
|
259
|
+
pending_passes = []
|
|
260
|
+
|
|
261
|
+
for line in lines:
|
|
262
|
+
stripped = line.strip()
|
|
263
|
+
if _PASS_RE.search(stripped) and not _ERROR_RE.search(stripped):
|
|
264
|
+
pending_passes.append(line)
|
|
265
|
+
continue
|
|
266
|
+
flush_pending()
|
|
267
|
+
result.append(line)
|
|
268
|
+
|
|
269
|
+
flush_pending()
|
|
270
|
+
return result
|
|
271
|
+
|
|
272
|
+
def _collapse_repeats(self, lines: list[str]) -> list[str]:
|
|
273
|
+
"""Replace runs of identical lines with [line repeated xN]."""
|
|
274
|
+
if not lines:
|
|
275
|
+
return lines
|
|
276
|
+
|
|
277
|
+
result = []
|
|
278
|
+
prev = lines[0]
|
|
279
|
+
count = 1
|
|
280
|
+
|
|
281
|
+
for line in lines[1:]:
|
|
282
|
+
stripped = line.strip()
|
|
283
|
+
if stripped == prev.strip() and stripped:
|
|
284
|
+
count += 1
|
|
285
|
+
else:
|
|
286
|
+
if count > 2:
|
|
287
|
+
result.append(prev)
|
|
288
|
+
result.append(f"[line repeated x{count}]")
|
|
289
|
+
else:
|
|
290
|
+
result.extend([prev] * count)
|
|
291
|
+
prev = line
|
|
292
|
+
count = 1
|
|
293
|
+
|
|
294
|
+
if count > 2:
|
|
295
|
+
result.append(prev)
|
|
296
|
+
result.append(f"[line repeated x{count}]")
|
|
297
|
+
else:
|
|
298
|
+
result.extend([prev] * count)
|
|
299
|
+
|
|
300
|
+
return result
|
|
301
|
+
|
|
302
|
+
def _collapse_global_repeats(self, lines: list[str], mode: str = "generic") -> list[str]:
|
|
303
|
+
"""Summarize noisy repeated lines even when they are not consecutive."""
|
|
304
|
+
if not lines:
|
|
305
|
+
return lines
|
|
306
|
+
|
|
307
|
+
normalized = [line.strip() for line in lines if line.strip()]
|
|
308
|
+
counts = Counter(normalized)
|
|
309
|
+
thresholds = {"test": 2, "install": 2, "build": 3, "generic": 4}
|
|
310
|
+
threshold = thresholds.get(mode, 4)
|
|
311
|
+
noisy = {
|
|
312
|
+
line for line, count in counts.items()
|
|
313
|
+
if count >= threshold and not _ERROR_RE.search(line) and len(line) <= 180
|
|
314
|
+
}
|
|
315
|
+
if not noisy:
|
|
316
|
+
return lines
|
|
317
|
+
|
|
318
|
+
result = []
|
|
319
|
+
emitted = set()
|
|
320
|
+
for line in lines:
|
|
321
|
+
stripped = line.strip()
|
|
322
|
+
if stripped in noisy:
|
|
323
|
+
if stripped in emitted:
|
|
324
|
+
continue
|
|
325
|
+
emitted.add(stripped)
|
|
326
|
+
result.append(line)
|
|
327
|
+
result.append(f"[line repeated x{counts[stripped]} across output]")
|
|
328
|
+
else:
|
|
329
|
+
result.append(line)
|
|
330
|
+
return result
|
|
331
|
+
|
|
332
|
+
def _collapse_tracebacks(self, lines: list[str]) -> list[str]:
|
|
333
|
+
"""Condense traceback blocks to the signal-bearing lines."""
|
|
334
|
+
result = []
|
|
335
|
+
i = 0
|
|
336
|
+
while i < len(lines):
|
|
337
|
+
line = lines[i]
|
|
338
|
+
if "Traceback" not in line:
|
|
339
|
+
result.append(line)
|
|
340
|
+
i += 1
|
|
341
|
+
continue
|
|
342
|
+
|
|
343
|
+
block = [line]
|
|
344
|
+
i += 1
|
|
345
|
+
while i < len(lines):
|
|
346
|
+
current = lines[i]
|
|
347
|
+
if not current.strip():
|
|
348
|
+
block.append(current)
|
|
349
|
+
i += 1
|
|
350
|
+
break
|
|
351
|
+
if _ERROR_RE.search(current) or current.lstrip().startswith("File "):
|
|
352
|
+
block.append(current)
|
|
353
|
+
i += 1
|
|
354
|
+
continue
|
|
355
|
+
if current.startswith(" ") or current.startswith("\t"):
|
|
356
|
+
block.append(current)
|
|
357
|
+
i += 1
|
|
358
|
+
continue
|
|
359
|
+
break
|
|
360
|
+
|
|
361
|
+
file_lines = [b.strip() for b in block if b.strip().startswith("File ")]
|
|
362
|
+
tail = next((b.strip() for b in reversed(block) if b.strip() and "Traceback" not in b.strip()), "")
|
|
363
|
+
result.append("[traceback]")
|
|
364
|
+
if file_lines:
|
|
365
|
+
result.append(file_lines[0])
|
|
366
|
+
if len(file_lines) > 1:
|
|
367
|
+
result.append(f"[{len(file_lines) - 1} more stack frames]")
|
|
368
|
+
if tail:
|
|
369
|
+
result.append(tail)
|
|
370
|
+
|
|
371
|
+
return result
|
|
372
|
+
|
|
373
|
+
def _normalize_blanks(self, lines: list[str]) -> list[str]:
|
|
374
|
+
"""Collapse multiple consecutive blank lines to one."""
|
|
375
|
+
result = []
|
|
376
|
+
prev_blank = False
|
|
377
|
+
for line in lines:
|
|
378
|
+
is_blank = not line.strip()
|
|
379
|
+
if is_blank:
|
|
380
|
+
if not prev_blank:
|
|
381
|
+
result.append(line)
|
|
382
|
+
prev_blank = True
|
|
383
|
+
else:
|
|
384
|
+
prev_blank = False
|
|
385
|
+
result.append(line)
|
|
386
|
+
return result
|
|
387
|
+
|
|
388
|
+
# ── Pass 2: LLM summarization ────────────────────────
|
|
389
|
+
|
|
390
|
+
def _summarize_signal_output(self, text: str, mode: str = "generic") -> str:
|
|
391
|
+
"""Shrink long output to signal-bearing lines while preserving failures."""
|
|
392
|
+
lines = [line.rstrip() for line in text.splitlines()]
|
|
393
|
+
nonblank = [line for line in lines if line.strip()]
|
|
394
|
+
if len(nonblank) <= 12:
|
|
395
|
+
return text
|
|
396
|
+
|
|
397
|
+
max_lines = {"test": 5, "install": 6, "build": 6, "generic": 6}.get(mode, 6)
|
|
398
|
+
kept: list[str] = []
|
|
399
|
+
seen: set[str] = set()
|
|
400
|
+
has_errors = any(_ERROR_RE.search(line) for line in nonblank)
|
|
401
|
+
|
|
402
|
+
def add(line: str) -> None:
|
|
403
|
+
stripped = line.strip()
|
|
404
|
+
if not stripped or stripped in seen:
|
|
405
|
+
return
|
|
406
|
+
seen.add(stripped)
|
|
407
|
+
kept.append(line)
|
|
408
|
+
|
|
409
|
+
for line in nonblank[:3]:
|
|
410
|
+
stripped = line.strip()
|
|
411
|
+
if _PASS_RE.search(stripped) and not _WARNING_RE.search(stripped) and not _ERROR_RE.search(stripped):
|
|
412
|
+
continue
|
|
413
|
+
add(line)
|
|
414
|
+
if len(kept) >= 2:
|
|
415
|
+
break
|
|
416
|
+
|
|
417
|
+
if has_errors:
|
|
418
|
+
warning_lines = [line for line in nonblank if _WARNING_RE.search(line)]
|
|
419
|
+
error_lines = [line for line in nonblank if _ERROR_RE.search(line)]
|
|
420
|
+
repeat_lines = [line for line in nonblank if "[line repeated x" in line]
|
|
421
|
+
final_summary = next((line for line in reversed(nonblank) if _SUCCESS_SUMMARY_RE.search(line)), "")
|
|
422
|
+
for line in repeat_lines[-2:]:
|
|
423
|
+
add(line)
|
|
424
|
+
if warning_lines:
|
|
425
|
+
add(f"[{len(warning_lines)} warning lines retained in mixed output]")
|
|
426
|
+
for line in warning_lines[-2:]:
|
|
427
|
+
add(line)
|
|
428
|
+
for line in error_lines[-max_lines:]:
|
|
429
|
+
add(line)
|
|
430
|
+
if final_summary:
|
|
431
|
+
add(final_summary)
|
|
432
|
+
else:
|
|
433
|
+
summary_lines = [
|
|
434
|
+
line for line in nonblank
|
|
435
|
+
if _SUCCESS_SUMMARY_RE.search(line) or _WARNING_RE.search(line)
|
|
436
|
+
]
|
|
437
|
+
for line in summary_lines[-max_lines:]:
|
|
438
|
+
add(line)
|
|
439
|
+
if not has_errors and not any(_SUCCESS_SUMMARY_RE.search(line) for line in kept):
|
|
440
|
+
for line in nonblank[-2:]:
|
|
441
|
+
add(line)
|
|
442
|
+
|
|
443
|
+
omitted = len(nonblank) - len(kept)
|
|
444
|
+
if omitted > 0:
|
|
445
|
+
label = "non-error" if has_errors else "successful"
|
|
446
|
+
kept.insert(min(2, len(kept)), f"[{omitted} {label} lines omitted]")
|
|
447
|
+
|
|
448
|
+
return "\n".join(kept) if kept else text
|
|
449
|
+
|
|
450
|
+
def _pass2(self, filtered_text: str, raw_text: str = "") -> str | None:
|
|
451
|
+
"""Use Ollama LLM to generate a 3-5 line summary of terminal output.
|
|
452
|
+
|
|
453
|
+
Status-aware: on success, be very terse. On failure, preserve error details.
|
|
454
|
+
"""
|
|
455
|
+
has_errors = bool(_ERROR_RE.search(filtered_text))
|
|
456
|
+
|
|
457
|
+
if has_errors:
|
|
458
|
+
system = (
|
|
459
|
+
"You are a terminal output summarizer. The output contains errors or failures. "
|
|
460
|
+
"Summarize in 3-5 lines. PRESERVE all error messages, file paths, and line numbers. "
|
|
461
|
+
"Start with the failure count/type."
|
|
462
|
+
)
|
|
463
|
+
else:
|
|
464
|
+
system = (
|
|
465
|
+
"You are a terminal output summarizer. The output shows success. "
|
|
466
|
+
"Summarize in 1-3 lines. Include: what ran, result counts, duration if shown. "
|
|
467
|
+
"Be extremely terse."
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
# Smart truncation: keep head (context) and tail (usually contains the final error/summary)
|
|
471
|
+
if len(filtered_text) > 4000:
|
|
472
|
+
smart_context = filtered_text[:1000] + "\n... [TRUNCATED C3 FILTER] ...\n" + filtered_text[-3000:]
|
|
473
|
+
else:
|
|
474
|
+
smart_context = filtered_text
|
|
475
|
+
|
|
476
|
+
prompt = f"Summarize this terminal output:\n\n{smart_context}"
|
|
477
|
+
|
|
478
|
+
result = self.ollama.generate(
|
|
479
|
+
prompt=prompt,
|
|
480
|
+
model=self.filter_model,
|
|
481
|
+
system=system,
|
|
482
|
+
temperature=0.1,
|
|
483
|
+
max_tokens=200,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
if result and count_tokens(result) < count_tokens(filtered_text):
|
|
487
|
+
return f"[c3:filter:llm] {result.strip()}"
|
|
488
|
+
return None
|