brainlayer 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- brainlayer/__init__.py +3 -0
- brainlayer/cli/__init__.py +1545 -0
- brainlayer/cli/wizard.py +132 -0
- brainlayer/cli_new.py +151 -0
- brainlayer/client.py +164 -0
- brainlayer/clustering.py +736 -0
- brainlayer/daemon.py +1105 -0
- brainlayer/dashboard/README.md +129 -0
- brainlayer/dashboard/__init__.py +5 -0
- brainlayer/dashboard/app.py +151 -0
- brainlayer/dashboard/search.py +229 -0
- brainlayer/dashboard/views.py +230 -0
- brainlayer/embeddings.py +131 -0
- brainlayer/engine.py +550 -0
- brainlayer/index_new.py +87 -0
- brainlayer/mcp/__init__.py +1558 -0
- brainlayer/migrate.py +205 -0
- brainlayer/paths.py +43 -0
- brainlayer/pipeline/__init__.py +47 -0
- brainlayer/pipeline/analyze_communication.py +508 -0
- brainlayer/pipeline/brain_graph.py +567 -0
- brainlayer/pipeline/chat_tags.py +63 -0
- brainlayer/pipeline/chunk.py +422 -0
- brainlayer/pipeline/classify.py +472 -0
- brainlayer/pipeline/cluster_sampling.py +73 -0
- brainlayer/pipeline/enrichment.py +810 -0
- brainlayer/pipeline/extract.py +66 -0
- brainlayer/pipeline/extract_claude_desktop.py +149 -0
- brainlayer/pipeline/extract_corrections.py +231 -0
- brainlayer/pipeline/extract_markdown.py +195 -0
- brainlayer/pipeline/extract_whatsapp.py +227 -0
- brainlayer/pipeline/git_overlay.py +301 -0
- brainlayer/pipeline/longitudinal_analyzer.py +568 -0
- brainlayer/pipeline/obsidian_export.py +455 -0
- brainlayer/pipeline/operation_grouping.py +486 -0
- brainlayer/pipeline/plan_linking.py +313 -0
- brainlayer/pipeline/sanitize.py +549 -0
- brainlayer/pipeline/semantic_style.py +574 -0
- brainlayer/pipeline/session_enrichment.py +472 -0
- brainlayer/pipeline/style_embed.py +67 -0
- brainlayer/pipeline/style_index.py +139 -0
- brainlayer/pipeline/temporal_chains.py +203 -0
- brainlayer/pipeline/time_batcher.py +248 -0
- brainlayer/pipeline/unified_timeline.py +569 -0
- brainlayer/storage.py +66 -0
- brainlayer/store.py +155 -0
- brainlayer/taxonomy.json +80 -0
- brainlayer/vector_store.py +1891 -0
- brainlayer-1.0.0.dist-info/METADATA +313 -0
- brainlayer-1.0.0.dist-info/RECORD +53 -0
- brainlayer-1.0.0.dist-info/WHEEL +4 -0
- brainlayer-1.0.0.dist-info/entry_points.txt +4 -0
- brainlayer-1.0.0.dist-info/licenses/LICENSE +190 -0
|
@@ -0,0 +1,1545 @@
|
|
|
1
|
+
"""BrainLayer CLI - Command line interface for the knowledge pipeline."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re as _re
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from rich import print as rprint
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
from rich.progress import (
|
|
13
|
+
BarColumn,
|
|
14
|
+
MofNCompleteColumn,
|
|
15
|
+
Progress,
|
|
16
|
+
SpinnerColumn,
|
|
17
|
+
TaskProgressColumn,
|
|
18
|
+
TextColumn,
|
|
19
|
+
TimeElapsedColumn,
|
|
20
|
+
TimeRemainingColumn,
|
|
21
|
+
)
|
|
22
|
+
from rich.table import Table
|
|
23
|
+
|
|
24
|
+
app = typer.Typer(
|
|
25
|
+
name="brainlayer",
|
|
26
|
+
help="זיכרון - Local knowledge pipeline for Claude Code conversations",
|
|
27
|
+
no_args_is_help=True,
|
|
28
|
+
)
|
|
29
|
+
console = Console()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@app.command()
|
|
33
|
+
def init() -> None:
|
|
34
|
+
"""Interactive setup wizard — detects your environment and configures BrainLayer."""
|
|
35
|
+
from .wizard import run_wizard
|
|
36
|
+
|
|
37
|
+
run_wizard()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@app.command()
|
|
41
|
+
def index(
|
|
42
|
+
source: Path = typer.Argument(
|
|
43
|
+
Path.home() / ".claude" / "projects", help="Source directory containing JSONL conversations"
|
|
44
|
+
),
|
|
45
|
+
project: str = typer.Option(None, "--project", "-p", help="Only index specific project (folder name)"),
|
|
46
|
+
force: bool = typer.Option(False, "--force", "-f", help="Re-index all files (ignore cache)"),
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Index Claude Code conversations into the knowledge base (sqlite-vec)."""
|
|
49
|
+
# Delegate to the sqlite-vec index implementation
|
|
50
|
+
index_fast(source, project, force)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@app.command()
|
|
54
|
+
def dashboard() -> None:
|
|
55
|
+
"""Launch interactive dashboard for memory search and management."""
|
|
56
|
+
try:
|
|
57
|
+
from ..dashboard import DashboardApp
|
|
58
|
+
|
|
59
|
+
app = DashboardApp()
|
|
60
|
+
app.run()
|
|
61
|
+
|
|
62
|
+
except ImportError as e:
|
|
63
|
+
rprint(f"[red]Dashboard dependencies missing: {e}[/]")
|
|
64
|
+
rprint("[yellow]Install with: pip install -e .[dev][/]")
|
|
65
|
+
except Exception as e:
|
|
66
|
+
rprint(f"[red]Dashboard error: {e}[/]")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@app.command()
|
|
70
|
+
def search(
|
|
71
|
+
query: str = typer.Argument(..., help="Search query"),
|
|
72
|
+
n: int = typer.Option(5, "--num", "-n", help="Number of results", min=1, max=100),
|
|
73
|
+
project: str = typer.Option(None, "--project", "-p", help="Filter by project"),
|
|
74
|
+
content_type: str = typer.Option(None, "--type", "-t", help="Filter by content type"),
|
|
75
|
+
text: bool = typer.Option(False, "--text", help="Use text-based search instead of semantic search"),
|
|
76
|
+
hybrid: bool = typer.Option(
|
|
77
|
+
True, "--hybrid/--no-hybrid", help="Use hybrid search (semantic + keyword). Default: hybrid"
|
|
78
|
+
),
|
|
79
|
+
) -> None:
|
|
80
|
+
"""Search the knowledge base (sqlite-vec). Uses hybrid search by default."""
|
|
81
|
+
from ..cli_new import search_command
|
|
82
|
+
|
|
83
|
+
search_command(query, n, project, content_type, text, hybrid)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@app.command()
|
|
87
|
+
def context(
|
|
88
|
+
chunk_id: str = typer.Argument(..., help="Chunk ID from a search result"),
|
|
89
|
+
before: int = typer.Option(3, "--before", "-b", help="Chunks before target"),
|
|
90
|
+
after: int = typer.Option(3, "--after", "-a", help="Chunks after target"),
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Show surrounding conversation context for a search result."""
|
|
93
|
+
try:
|
|
94
|
+
from ..client import get_client
|
|
95
|
+
|
|
96
|
+
rprint(f"[bold blue]זיכרון[/] - Context for chunk: [dim]{chunk_id[:40]}...[/]")
|
|
97
|
+
|
|
98
|
+
client = get_client()
|
|
99
|
+
|
|
100
|
+
with console.status("[bold green]Fetching context..."):
|
|
101
|
+
result = client.get_context(chunk_id, before=before, after=after)
|
|
102
|
+
|
|
103
|
+
if not result.get("context"):
|
|
104
|
+
rprint("[yellow]No context available for this chunk.[/]")
|
|
105
|
+
return
|
|
106
|
+
|
|
107
|
+
for chunk in result["context"]:
|
|
108
|
+
marker = " [bold green]<< TARGET >>[/]" if chunk.get("is_target") else ""
|
|
109
|
+
ctype = chunk.get("content_type", "unknown")
|
|
110
|
+
pos = chunk.get("position", "?")
|
|
111
|
+
rprint(f"\n[bold cyan]Position {pos}[/] [dim]({ctype})[/]{marker}")
|
|
112
|
+
content = chunk.get("content", "")
|
|
113
|
+
rprint(content[:1500] + ("..." if len(content) > 1500 else ""))
|
|
114
|
+
rprint("[dim]---[/]")
|
|
115
|
+
|
|
116
|
+
except Exception as e:
|
|
117
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
118
|
+
raise typer.Exit(1)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# Known project renames/aliases - map old names to canonical names
|
|
122
|
+
# Example: pre-monorepo standalone repos → monorepo
|
|
123
|
+
PROJECT_ALIASES: dict[str, str] = {
|
|
124
|
+
# Add your project aliases here.
|
|
125
|
+
# Example: "my-old-repo": "my-project",
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
# Sub-package prefixes that should resolve to parent repo
|
|
129
|
+
# Example: "my-project-packages-api": "my-project"
|
|
130
|
+
_MONOREPO_PACKAGES: dict[str, str] = {
|
|
131
|
+
"my-project-api": "my-project",
|
|
132
|
+
"rudy-monorepo-apps-jem": "rudy-monorepo",
|
|
133
|
+
"app-b-apps-public": "app-b",
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
# Regex for worktree/nightshift suffixes
|
|
137
|
+
_WORKTREE_SUFFIX = _re.compile(r"^(.+?)(?:-nightshift-\d+|-worktrees-.+|-haiku)$")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _normalize_project_name(raw_name: str) -> str:
|
|
141
|
+
"""Normalize a raw project folder name to a clean canonical name.
|
|
142
|
+
|
|
143
|
+
Used during indexing to store clean names in the database.
|
|
144
|
+
Handles: path prefixes, sub-packages, worktree suffixes, aliases.
|
|
145
|
+
"""
|
|
146
|
+
# First clean the path-style name (strips -Users-username-Gits- etc.)
|
|
147
|
+
cleaned = _clean_project_name(raw_name)
|
|
148
|
+
|
|
149
|
+
# Strip worktree/nightshift suffixes
|
|
150
|
+
m = _WORKTREE_SUFFIX.match(cleaned)
|
|
151
|
+
if m:
|
|
152
|
+
cleaned = m.group(1)
|
|
153
|
+
|
|
154
|
+
# Check sub-package mappings
|
|
155
|
+
if cleaned in _MONOREPO_PACKAGES:
|
|
156
|
+
return _MONOREPO_PACKAGES[cleaned]
|
|
157
|
+
|
|
158
|
+
# Check aliases (pre-monorepo standalone repos)
|
|
159
|
+
if cleaned in PROJECT_ALIASES:
|
|
160
|
+
return PROJECT_ALIASES[cleaned]
|
|
161
|
+
|
|
162
|
+
return cleaned
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@app.command()
|
|
166
|
+
def review(
|
|
167
|
+
limit: int = typer.Option(20, "--limit", "-n", help="Number of low-confidence chunks to review"),
|
|
168
|
+
threshold: float = typer.Option(0.6, "--threshold", "-t", help="Confidence threshold"),
|
|
169
|
+
) -> None:
|
|
170
|
+
"""Review low-confidence auto-tagged chunks and correct labels."""
|
|
171
|
+
try:
|
|
172
|
+
import json
|
|
173
|
+
from pathlib import Path
|
|
174
|
+
|
|
175
|
+
from ..vector_store import VectorStore
|
|
176
|
+
|
|
177
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
178
|
+
store = VectorStore(db_path)
|
|
179
|
+
cursor = store.conn.cursor()
|
|
180
|
+
|
|
181
|
+
# Count tagged chunks
|
|
182
|
+
total = store.count()
|
|
183
|
+
tagged = list(cursor.execute("SELECT COUNT(*) FROM chunks WHERE tags IS NOT NULL AND tags != '[]'"))[0][0]
|
|
184
|
+
low_conf = list(
|
|
185
|
+
cursor.execute(
|
|
186
|
+
"SELECT COUNT(*) FROM chunks WHERE tag_confidence IS NOT NULL AND tag_confidence < ?",
|
|
187
|
+
(threshold,),
|
|
188
|
+
)
|
|
189
|
+
)[0][0]
|
|
190
|
+
|
|
191
|
+
rprint("[bold blue]זיכרון[/] - Tag Review Queue")
|
|
192
|
+
rprint(f"Total: {total:,} | Tagged: {tagged:,} | Low confidence (<{threshold}): {low_conf:,}")
|
|
193
|
+
|
|
194
|
+
if tagged == 0:
|
|
195
|
+
rprint("[yellow]No chunks tagged yet. Run: python scripts/classify-all.py[/]")
|
|
196
|
+
return
|
|
197
|
+
|
|
198
|
+
# Show worst chunks
|
|
199
|
+
rows = list(
|
|
200
|
+
cursor.execute(
|
|
201
|
+
"""
|
|
202
|
+
SELECT id, content, tags, tag_confidence, project, content_type
|
|
203
|
+
FROM chunks
|
|
204
|
+
WHERE tag_confidence IS NOT NULL AND tag_confidence < ?
|
|
205
|
+
ORDER BY tag_confidence ASC
|
|
206
|
+
LIMIT ?
|
|
207
|
+
""",
|
|
208
|
+
(threshold, limit),
|
|
209
|
+
)
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
if not rows:
|
|
213
|
+
rprint(f"[green]No chunks below {threshold} confidence![/]")
|
|
214
|
+
return
|
|
215
|
+
|
|
216
|
+
for i, row in enumerate(rows):
|
|
217
|
+
chunk_id, content, tags_json, conf, proj, ct = row
|
|
218
|
+
tags = json.loads(tags_json) if tags_json else []
|
|
219
|
+
rprint(f"\n[bold cyan]{i + 1}.[/] [dim]({proj}/{ct})[/] conf={conf:.2f}")
|
|
220
|
+
rprint(f" Tags: {', '.join(tags) if tags else '[none]'}")
|
|
221
|
+
rprint(f" Content: {content[:200]}...")
|
|
222
|
+
rprint(f" [dim]ID: {chunk_id}[/]")
|
|
223
|
+
|
|
224
|
+
rprint("\n[dim]Use label-chunks.py for interactive correction, then retrain.[/]")
|
|
225
|
+
store.close()
|
|
226
|
+
|
|
227
|
+
except Exception as e:
|
|
228
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
229
|
+
raise typer.Exit(1)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _clean_project_name(name: str) -> str:
|
|
233
|
+
"""Clean up project names by extracting the repo name from path-style names.
|
|
234
|
+
|
|
235
|
+
Examples:
|
|
236
|
+
-Users-username-Gits-my-project -> my-project
|
|
237
|
+
-Users-username-Desktop-Gits-another-project -> another-project
|
|
238
|
+
-Users-username-Gits-my-site -> my-site
|
|
239
|
+
-Users-username--config-myapp -> myapp
|
|
240
|
+
"""
|
|
241
|
+
if not name.startswith("-Users-") and not name.startswith("-home-"):
|
|
242
|
+
return name
|
|
243
|
+
|
|
244
|
+
parts = name.split("-")
|
|
245
|
+
|
|
246
|
+
# Find index of "Gits" or last occurrence of path markers
|
|
247
|
+
markers = {"Gits", "Desktop", "projects", "config"}
|
|
248
|
+
last_marker_idx = -1
|
|
249
|
+
for i, part in enumerate(parts):
|
|
250
|
+
if part in markers:
|
|
251
|
+
last_marker_idx = i
|
|
252
|
+
|
|
253
|
+
if last_marker_idx >= 0 and last_marker_idx < len(parts) - 1:
|
|
254
|
+
# Take everything after the last marker
|
|
255
|
+
repo_parts = [p for p in parts[last_marker_idx + 1 :] if p]
|
|
256
|
+
if repo_parts:
|
|
257
|
+
return "-".join(repo_parts)
|
|
258
|
+
|
|
259
|
+
# Fallback: filter out common path parts and join remaining
|
|
260
|
+
skip = {"Users", "home", "Desktop", "Gits", "projects", "config"}
|
|
261
|
+
meaningful = [p for p in parts if p and p not in skip]
|
|
262
|
+
if meaningful:
|
|
263
|
+
return "-".join(meaningful[-2:]) if len(meaningful) > 1 else meaningful[-1]
|
|
264
|
+
|
|
265
|
+
return name
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
@app.command()
|
|
269
|
+
def stats() -> None:
|
|
270
|
+
"""Show knowledge base statistics (sqlite-vec)."""
|
|
271
|
+
from ..cli_new import stats_command
|
|
272
|
+
|
|
273
|
+
stats_command()
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
@app.command()
|
|
277
|
+
def clear(confirm: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation")) -> None:
|
|
278
|
+
"""Clear the entire knowledge base."""
|
|
279
|
+
try:
|
|
280
|
+
from pathlib import Path
|
|
281
|
+
|
|
282
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
283
|
+
|
|
284
|
+
if not confirm:
|
|
285
|
+
confirm = typer.confirm("Are you sure you want to clear the knowledge base?")
|
|
286
|
+
|
|
287
|
+
if confirm:
|
|
288
|
+
if db_path.exists():
|
|
289
|
+
db_path.unlink()
|
|
290
|
+
for suffix in ["-shm", "-wal"]:
|
|
291
|
+
p = db_path.parent / (db_path.name + suffix)
|
|
292
|
+
if p.exists():
|
|
293
|
+
p.unlink()
|
|
294
|
+
rprint("[bold green]✓[/] Knowledge base cleared")
|
|
295
|
+
else:
|
|
296
|
+
rprint("[yellow]Cancelled[/]")
|
|
297
|
+
|
|
298
|
+
except Exception as e:
|
|
299
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
300
|
+
raise typer.Exit(1)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@app.command()
|
|
304
|
+
def analyze_style(
|
|
305
|
+
whatsapp_limit: int = typer.Option(1000, "--whatsapp-limit", "-w", help="Number of WhatsApp messages to analyze"),
|
|
306
|
+
claude_export: Path = typer.Option(
|
|
307
|
+
None, "--claude-export", "-c", help="Path to Claude chat export JSON (optional)"
|
|
308
|
+
),
|
|
309
|
+
output: Path = typer.Option(
|
|
310
|
+
Path.home() / ".cursor/rules/communication-style.md",
|
|
311
|
+
"--output",
|
|
312
|
+
"-o",
|
|
313
|
+
help="Output path for generated rules",
|
|
314
|
+
),
|
|
315
|
+
export_analysis: bool = typer.Option(False, "--export-json", help="Also export full analysis as JSON"),
|
|
316
|
+
) -> None:
|
|
317
|
+
"""Analyze communication patterns from WhatsApp and Claude chats.
|
|
318
|
+
|
|
319
|
+
Extracts your writing style and Claude's response patterns to generate
|
|
320
|
+
personalized rules for desktop apps that help with text interactions.
|
|
321
|
+
"""
|
|
322
|
+
try:
|
|
323
|
+
from ..pipeline.analyze_communication import CommunicationAnalyzer
|
|
324
|
+
from ..pipeline.extract_claude_desktop import extract_claude_chats
|
|
325
|
+
from ..pipeline.extract_whatsapp import analyze_writing_style, extract_whatsapp_messages
|
|
326
|
+
|
|
327
|
+
rprint("[bold blue]זיכרון[/] - Analyzing communication patterns\n")
|
|
328
|
+
|
|
329
|
+
analyzer = CommunicationAnalyzer()
|
|
330
|
+
|
|
331
|
+
# Extract WhatsApp messages
|
|
332
|
+
with console.status("[bold green]Extracting WhatsApp messages..."):
|
|
333
|
+
try:
|
|
334
|
+
messages = list(
|
|
335
|
+
extract_whatsapp_messages(
|
|
336
|
+
limit=whatsapp_limit,
|
|
337
|
+
only_from_me=False, # Get both sides for context
|
|
338
|
+
exclude_groups=True,
|
|
339
|
+
)
|
|
340
|
+
)
|
|
341
|
+
analyzer.add_whatsapp_messages(messages)
|
|
342
|
+
rprint(f"[green]✓[/] Extracted {len(messages)} WhatsApp messages")
|
|
343
|
+
except FileNotFoundError as e:
|
|
344
|
+
rprint("[yellow]Warning:[/] WhatsApp database not found")
|
|
345
|
+
rprint(f"[dim] {e}[/]")
|
|
346
|
+
except Exception as e:
|
|
347
|
+
rprint(f"[yellow]Warning:[/] Failed to extract WhatsApp: {e}")
|
|
348
|
+
|
|
349
|
+
# Extract Claude chats if export provided
|
|
350
|
+
if claude_export and claude_export.exists():
|
|
351
|
+
with console.status("[bold green]Extracting Claude conversations..."):
|
|
352
|
+
try:
|
|
353
|
+
conversations = list(extract_claude_chats(method="manual"))
|
|
354
|
+
analyzer.add_claude_conversations(conversations)
|
|
355
|
+
rprint(f"[green]✓[/] Extracted {len(conversations)} Claude conversations")
|
|
356
|
+
except Exception as e:
|
|
357
|
+
rprint(f"[yellow]Warning:[/] Failed to extract Claude chats: {e}")
|
|
358
|
+
else:
|
|
359
|
+
rprint("[dim]No Claude export provided (use --claude-export)[/]")
|
|
360
|
+
rprint("[dim]To export from Claude.ai:[/]")
|
|
361
|
+
rprint("[dim] 1. Go to Settings > Data & Privacy[/]")
|
|
362
|
+
rprint("[dim] 2. Click 'Export my data'[/]")
|
|
363
|
+
rprint("[dim] 3. Pass the file with --claude-export PATH[/]\n")
|
|
364
|
+
|
|
365
|
+
# Check if we have enough data
|
|
366
|
+
if not analyzer.user_messages:
|
|
367
|
+
rprint("[bold red]Error:[/] No messages found to analyze")
|
|
368
|
+
rprint("Make sure WhatsApp database is accessible or provide Claude export")
|
|
369
|
+
raise typer.Exit(1)
|
|
370
|
+
|
|
371
|
+
# Generate analysis
|
|
372
|
+
with console.status("[bold green]Analyzing communication patterns..."):
|
|
373
|
+
writing_style = analyzer.analyze_writing_style()
|
|
374
|
+
response_patterns = analyzer.analyze_claude_response_patterns()
|
|
375
|
+
clarifications = analyzer.extract_common_clarifications()
|
|
376
|
+
|
|
377
|
+
# Display summary
|
|
378
|
+
rprint("\n[bold]Analysis Summary:[/]\n")
|
|
379
|
+
rprint(f"Messages analyzed: [bold]{writing_style.get('total_messages_analyzed', 0)}[/]")
|
|
380
|
+
rprint(f"Avg message length: [bold]{writing_style.get('avg_message_length', 0):.0f}[/] chars")
|
|
381
|
+
rprint(f"Formality score: [bold]{writing_style.get('formality_score', 0):.2f}[/] (0=informal, 1=formal)")
|
|
382
|
+
rprint(f"Emoji rate: [bold]{writing_style.get('emoji_rate', 0):.2f}[/] per message")
|
|
383
|
+
|
|
384
|
+
if response_patterns.get("total_responses_analyzed", 0) > 0:
|
|
385
|
+
rprint(f"\nClaude responses analyzed: [bold]{response_patterns['total_responses_analyzed']}[/]")
|
|
386
|
+
rprint(f"Common clarifications found: [bold]{len(clarifications)}[/]")
|
|
387
|
+
|
|
388
|
+
# Generate rules
|
|
389
|
+
with console.status("[bold green]Generating rules..."):
|
|
390
|
+
rules = analyzer.generate_rules(output)
|
|
391
|
+
|
|
392
|
+
rprint(f"\n[bold green]✓[/] Rules generated: [cyan]{output}[/]")
|
|
393
|
+
|
|
394
|
+
# Export full analysis if requested
|
|
395
|
+
if export_analysis:
|
|
396
|
+
json_path = output.with_suffix(".json")
|
|
397
|
+
analyzer.export_analysis(json_path)
|
|
398
|
+
rprint(f"[bold green]✓[/] Analysis exported: [cyan]{json_path}[/]")
|
|
399
|
+
|
|
400
|
+
# Show preview
|
|
401
|
+
rprint("\n[bold]Preview of generated rules:[/]\n")
|
|
402
|
+
preview_lines = rules.split("\n")[:25]
|
|
403
|
+
for line in preview_lines:
|
|
404
|
+
rprint(f"[dim]{line}[/]")
|
|
405
|
+
total_lines = len(rules.split("\n"))
|
|
406
|
+
if total_lines > 25:
|
|
407
|
+
rprint(f"[dim]... ({total_lines - 25} more lines)[/]")
|
|
408
|
+
|
|
409
|
+
except typer.Exit:
|
|
410
|
+
raise
|
|
411
|
+
except Exception as e:
|
|
412
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
413
|
+
import traceback
|
|
414
|
+
|
|
415
|
+
traceback.print_exc()
|
|
416
|
+
raise typer.Exit(1)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
@app.command("list-chats")
|
|
420
|
+
def list_chats(
|
|
421
|
+
output: Path = typer.Option(None, "--output", "-o", help="Save to CSV file (default: print only)"),
|
|
422
|
+
source: str = typer.Option("whatsapp", "--source", "-s", help="Filter by source: whatsapp, claude, or all"),
|
|
423
|
+
claude_export: Path = typer.Option(
|
|
424
|
+
None, "--claude-export", "-c", help="Path to Claude export (for source=claude or all)"
|
|
425
|
+
),
|
|
426
|
+
) -> None:
|
|
427
|
+
"""List chats with message counts for tagging (family, friends, co-workers)."""
|
|
428
|
+
try:
|
|
429
|
+
from ..pipeline.unified_timeline import UnifiedTimeline
|
|
430
|
+
|
|
431
|
+
rprint("[bold blue]זיכרון[/] - Listing chats\n")
|
|
432
|
+
|
|
433
|
+
timeline = UnifiedTimeline()
|
|
434
|
+
src_filter = None if source == "all" else source
|
|
435
|
+
|
|
436
|
+
if source in ("whatsapp", "all"):
|
|
437
|
+
with console.status("[bold green]Loading WhatsApp..."):
|
|
438
|
+
try:
|
|
439
|
+
timeline.load_whatsapp()
|
|
440
|
+
except FileNotFoundError as e:
|
|
441
|
+
rprint(f"[yellow]WhatsApp:[/] {e}")
|
|
442
|
+
|
|
443
|
+
if source in ("claude", "all"):
|
|
444
|
+
claude_path = claude_export or Path("/tmp/claude-export/conversations.json")
|
|
445
|
+
if claude_path.exists():
|
|
446
|
+
with console.status("[bold green]Loading Claude..."):
|
|
447
|
+
timeline.load_claude(claude_path)
|
|
448
|
+
else:
|
|
449
|
+
rprint("[yellow]Claude:[/] No export at /tmp/claude-export/")
|
|
450
|
+
|
|
451
|
+
chats = timeline.get_chat_list(source=src_filter)
|
|
452
|
+
if not chats:
|
|
453
|
+
rprint("[yellow]No chats found[/]")
|
|
454
|
+
raise typer.Exit(0)
|
|
455
|
+
|
|
456
|
+
# Print table
|
|
457
|
+
table = Table(title=f"Chats ({len(chats)} total)")
|
|
458
|
+
table.add_column("chat_id", style="dim")
|
|
459
|
+
table.add_column("contact_name")
|
|
460
|
+
table.add_column("messages", justify="right")
|
|
461
|
+
for chat_id, contact_name, count in chats[:30]:
|
|
462
|
+
table.add_row(
|
|
463
|
+
chat_id[:40] + "..." if len(str(chat_id)) > 43 else str(chat_id),
|
|
464
|
+
str(contact_name)[:30],
|
|
465
|
+
str(count),
|
|
466
|
+
)
|
|
467
|
+
if len(chats) > 30:
|
|
468
|
+
table.add_row("...", f"... and {len(chats) - 30} more", "")
|
|
469
|
+
console.print(table)
|
|
470
|
+
|
|
471
|
+
# Save CSV if requested
|
|
472
|
+
if output:
|
|
473
|
+
output = Path(output)
|
|
474
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
475
|
+
import csv
|
|
476
|
+
|
|
477
|
+
with open(output, "w", newline="", encoding="utf-8") as f:
|
|
478
|
+
w = csv.writer(f)
|
|
479
|
+
w.writerow(["chat_id", "contact_name", "message_count"])
|
|
480
|
+
w.writerows(chats)
|
|
481
|
+
rprint(f"\n[green]Saved to[/] {output}")
|
|
482
|
+
rprint("[dim]Add a 'tag' column (family|friends|co-workers) and use for chat-tags.yaml[/]")
|
|
483
|
+
|
|
484
|
+
except typer.Exit:
|
|
485
|
+
raise
|
|
486
|
+
except Exception as e:
|
|
487
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
488
|
+
raise typer.Exit(1)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
@app.command("analyze-evolution")
|
|
492
|
+
def analyze_evolution(
|
|
493
|
+
claude_export: Path = typer.Option(
|
|
494
|
+
None, "--claude-export", "-c", help="Path to Claude.ai export (conversations.json)"
|
|
495
|
+
),
|
|
496
|
+
instagram_export: Path = typer.Option(None, "--instagram", "-i", help="Path to Instagram export directory"),
|
|
497
|
+
gemini_export: Path = typer.Option(None, "--gemini", "-g", help="Path to Gemini/Google Takeout export"),
|
|
498
|
+
output_dir: Path = typer.Option(
|
|
499
|
+
Path("/tmp/style-analysis"), "--output", "-o", help="Output directory for analysis files"
|
|
500
|
+
),
|
|
501
|
+
granularity: str = typer.Option(
|
|
502
|
+
"half", "--granularity", help="Time period granularity: year, half, quarter, month"
|
|
503
|
+
),
|
|
504
|
+
model: str = typer.Option("qwen3-coder-64k", "--model", "-m", help="Ollama model for analysis"),
|
|
505
|
+
chat_tags: Path = typer.Option(
|
|
506
|
+
None,
|
|
507
|
+
"--chat-tags",
|
|
508
|
+
help="Path to chat-tags.yaml (default: ~/.config/brainlayer/chat-tags.yaml)",
|
|
509
|
+
),
|
|
510
|
+
use_embeddings: bool = typer.Option(
|
|
511
|
+
False,
|
|
512
|
+
"--use-embeddings",
|
|
513
|
+
"-e",
|
|
514
|
+
help="Use StyleDistance for style-aware cluster sampling (best for style analysis)",
|
|
515
|
+
),
|
|
516
|
+
yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt (for background/scripted runs)"),
|
|
517
|
+
) -> None:
|
|
518
|
+
"""Analyze communication style evolution over time.
|
|
519
|
+
|
|
520
|
+
Loads messages from WhatsApp, Claude, Instagram, and Gemini,
|
|
521
|
+
batches them by time period, analyzes each batch with LLM,
|
|
522
|
+
and generates evolution analysis with weighted master guide.
|
|
523
|
+
"""
|
|
524
|
+
try:
|
|
525
|
+
from ..pipeline.longitudinal_analyzer import run_full_analysis
|
|
526
|
+
from ..pipeline.time_batcher import create_time_batches, format_batches_summary
|
|
527
|
+
from ..pipeline.unified_timeline import UnifiedTimeline
|
|
528
|
+
|
|
529
|
+
rprint("[bold blue]זיכרון[/] - Longitudinal Style Analysis\n")
|
|
530
|
+
|
|
531
|
+
# Create unified timeline
|
|
532
|
+
timeline = UnifiedTimeline()
|
|
533
|
+
|
|
534
|
+
# Load WhatsApp
|
|
535
|
+
with console.status("[bold green]Loading WhatsApp messages..."):
|
|
536
|
+
try:
|
|
537
|
+
wa_count = timeline.load_whatsapp()
|
|
538
|
+
rprint(f"[green]✓[/] WhatsApp: {wa_count:,} messages")
|
|
539
|
+
except FileNotFoundError as e:
|
|
540
|
+
rprint(f"[yellow]⚠[/] WhatsApp not found: {e}")
|
|
541
|
+
|
|
542
|
+
# Load Claude if provided
|
|
543
|
+
if claude_export and claude_export.exists():
|
|
544
|
+
with console.status("[bold green]Loading Claude conversations..."):
|
|
545
|
+
try:
|
|
546
|
+
cl_count = timeline.load_claude(claude_export)
|
|
547
|
+
rprint(f"[green]✓[/] Claude: {cl_count:,} messages")
|
|
548
|
+
except Exception as e:
|
|
549
|
+
rprint(f"[yellow]⚠[/] Claude error: {e}")
|
|
550
|
+
|
|
551
|
+
# Load Instagram if provided
|
|
552
|
+
if instagram_export and instagram_export.exists():
|
|
553
|
+
with console.status("[bold green]Loading Instagram data..."):
|
|
554
|
+
try:
|
|
555
|
+
ig_count = timeline.load_instagram(instagram_export)
|
|
556
|
+
rprint(f"[green]✓[/] Instagram: {ig_count:,} messages")
|
|
557
|
+
except Exception as e:
|
|
558
|
+
rprint(f"[yellow]⚠[/] Instagram error: {e}")
|
|
559
|
+
|
|
560
|
+
# Load Gemini if provided or auto-detect in data/google-takeout
|
|
561
|
+
gemini_path = gemini_export
|
|
562
|
+
if not gemini_path or not gemini_path.exists():
|
|
563
|
+
takeout_dir = Path(__file__).resolve().parents[3] / "data" / "google-takeout"
|
|
564
|
+
if takeout_dir.exists():
|
|
565
|
+
for z in sorted(takeout_dir.glob("*.zip")):
|
|
566
|
+
if "-11-" in z.name:
|
|
567
|
+
gemini_path = z
|
|
568
|
+
break
|
|
569
|
+
if not gemini_path:
|
|
570
|
+
gemini_path = next(takeout_dir.glob("*.zip"), None)
|
|
571
|
+
if gemini_path and gemini_path.exists():
|
|
572
|
+
with console.status("[bold green]Loading Gemini data..."):
|
|
573
|
+
try:
|
|
574
|
+
ge_count = timeline.load_gemini(gemini_path)
|
|
575
|
+
rprint(f"[green]✓[/] Gemini: {ge_count:,} messages")
|
|
576
|
+
except Exception as e:
|
|
577
|
+
rprint(f"[yellow]⚠[/] Gemini error: {e}")
|
|
578
|
+
|
|
579
|
+
# Check if we have data
|
|
580
|
+
if not timeline.messages:
|
|
581
|
+
rprint("[bold red]Error:[/] No messages loaded")
|
|
582
|
+
raise typer.Exit(1)
|
|
583
|
+
|
|
584
|
+
# Apply chat tags if config exists
|
|
585
|
+
tagged = timeline.apply_chat_tags(chat_tags)
|
|
586
|
+
if tagged:
|
|
587
|
+
rprint(f"[green]✓[/] Applied relationship tags to {tagged:,} messages")
|
|
588
|
+
|
|
589
|
+
# Sort by time
|
|
590
|
+
timeline.sort_by_time()
|
|
591
|
+
|
|
592
|
+
# Embed and index if requested
|
|
593
|
+
style_collection = None
|
|
594
|
+
if use_embeddings:
|
|
595
|
+
from ..pipeline.style_embed import embed_messages, ensure_model
|
|
596
|
+
from ..pipeline.style_index import (
|
|
597
|
+
clear_style_collection,
|
|
598
|
+
get_style_client,
|
|
599
|
+
get_style_collection,
|
|
600
|
+
index_style_messages,
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
try:
|
|
604
|
+
ensure_model()
|
|
605
|
+
except ImportError as e:
|
|
606
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
607
|
+
rprint("[dim]Install: pip install sentence-transformers[/]")
|
|
608
|
+
raise typer.Exit(1)
|
|
609
|
+
|
|
610
|
+
def embed_progress(n, t):
|
|
611
|
+
if n > 0 and n % 5000 == 0:
|
|
612
|
+
rprint(f" [dim]Embedded {n:,} / {t:,}[/]")
|
|
613
|
+
|
|
614
|
+
with console.status("[bold green]Embedding messages with StyleDistance..."):
|
|
615
|
+
msg_embs = embed_messages(
|
|
616
|
+
timeline.messages,
|
|
617
|
+
on_progress=embed_progress,
|
|
618
|
+
)
|
|
619
|
+
rprint(f"[green]✓[/] Embedded {len(msg_embs):,} messages")
|
|
620
|
+
client = get_style_client()
|
|
621
|
+
coll = get_style_collection(client)
|
|
622
|
+
clear_style_collection(coll)
|
|
623
|
+
indexed = index_style_messages(msg_embs, coll)
|
|
624
|
+
rprint(f"[green]✓[/] Indexed {indexed:,} messages to vector store")
|
|
625
|
+
style_collection = coll
|
|
626
|
+
|
|
627
|
+
# Show stats
|
|
628
|
+
stats = timeline.get_stats()
|
|
629
|
+
rprint("\n[bold]Timeline Stats:[/]")
|
|
630
|
+
rprint(f" Total messages: {stats['total_messages']:,}")
|
|
631
|
+
rprint(f" Sources: {', '.join(stats['sources'])}")
|
|
632
|
+
rprint(f" Languages: {stats['by_language']}")
|
|
633
|
+
if stats["date_range"]:
|
|
634
|
+
rprint(f" Date range: {stats['date_range']['start'][:10]} to {stats['date_range']['end'][:10]}")
|
|
635
|
+
|
|
636
|
+
# Create time batches
|
|
637
|
+
rprint(f"\n[bold]Creating time batches (granularity: {granularity})...[/]")
|
|
638
|
+
batches = create_time_batches(timeline, granularity=granularity)
|
|
639
|
+
rprint(f"Created {len(batches)} batches")
|
|
640
|
+
|
|
641
|
+
# Show batch summary
|
|
642
|
+
rprint("\n" + format_batches_summary(batches))
|
|
643
|
+
|
|
644
|
+
# Confirm before running LLM analysis
|
|
645
|
+
rprint(f"\n[bold]Ready to analyze with {model}[/]")
|
|
646
|
+
rprint(f"This will take approximately {len(batches) * 2} minutes.")
|
|
647
|
+
|
|
648
|
+
if not yes and not typer.confirm("Continue with analysis?"):
|
|
649
|
+
rprint("[yellow]Cancelled[/]")
|
|
650
|
+
raise typer.Exit(0)
|
|
651
|
+
|
|
652
|
+
# Run full analysis
|
|
653
|
+
def progress_callback(msg, current, total):
|
|
654
|
+
rprint(f"[dim][{current + 1}/{total}] {msg}[/]")
|
|
655
|
+
|
|
656
|
+
rprint("\n[bold]Running analysis...[/]\n")
|
|
657
|
+
results = run_full_analysis(
|
|
658
|
+
batches,
|
|
659
|
+
output_dir,
|
|
660
|
+
languages=["hebrew", "english"],
|
|
661
|
+
model=model,
|
|
662
|
+
progress_callback=progress_callback,
|
|
663
|
+
style_collection=style_collection,
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
rprint("\n[bold green]✓ Analysis complete![/]")
|
|
667
|
+
rprint("\nOutput files:")
|
|
668
|
+
for name, path in results.items():
|
|
669
|
+
rprint(f" [cyan]{path}[/]")
|
|
670
|
+
|
|
671
|
+
# Show human summary
|
|
672
|
+
rprint("\n[bold]Human Summary:[/]\n")
|
|
673
|
+
summary_path = Path(results["summary_file"])
|
|
674
|
+
if summary_path.exists():
|
|
675
|
+
with open(summary_path) as f:
|
|
676
|
+
summary = f.read()
|
|
677
|
+
# Show first 30 lines
|
|
678
|
+
for line in summary.split("\n")[:30]:
|
|
679
|
+
rprint(f"[dim]{line}[/]")
|
|
680
|
+
|
|
681
|
+
except typer.Exit:
|
|
682
|
+
raise
|
|
683
|
+
except Exception as e:
|
|
684
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
685
|
+
import traceback
|
|
686
|
+
|
|
687
|
+
traceback.print_exc()
|
|
688
|
+
raise typer.Exit(1)
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
@app.command()
|
|
692
|
+
def serve() -> None:
|
|
693
|
+
"""Start the MCP server for Claude Code integration.
|
|
694
|
+
|
|
695
|
+
Note: MCP uses stdio (stdin/stdout), not network ports.
|
|
696
|
+
Configure in ~/.claude/settings.json under mcpServers.
|
|
697
|
+
"""
|
|
698
|
+
try:
|
|
699
|
+
from ..mcp import serve as mcp_serve
|
|
700
|
+
|
|
701
|
+
rprint("[bold blue]זיכרון[/] - Starting MCP server (stdio mode)")
|
|
702
|
+
mcp_serve()
|
|
703
|
+
|
|
704
|
+
except Exception as e:
|
|
705
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
706
|
+
raise typer.Exit(1)
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
@app.command("migrate")
|
|
710
|
+
def migrate() -> None:
|
|
711
|
+
"""Migrate from ChromaDB to sqlite-vec (one-time)."""
|
|
712
|
+
from ..cli_new import migrate_command
|
|
713
|
+
|
|
714
|
+
migrate_command()
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
@app.command("consolidate")
|
|
718
|
+
def consolidate(
|
|
719
|
+
execute: bool = typer.Option(False, "--execute", help="Apply changes (default: dry-run)"),
|
|
720
|
+
generate_rollback: bool = typer.Option(False, "--generate-rollback", help="Save rollback SQL"),
|
|
721
|
+
) -> None:
|
|
722
|
+
"""Consolidate fragmented project names into canonical names."""
|
|
723
|
+
import subprocess
|
|
724
|
+
|
|
725
|
+
script = Path(__file__).parent.parent.parent.parent / "scripts" / "consolidate_projects.py"
|
|
726
|
+
if not script.exists():
|
|
727
|
+
rprint(f"[bold red]Error:[/] Script not found: {script}")
|
|
728
|
+
raise typer.Exit(1)
|
|
729
|
+
args = [sys.executable, str(script)]
|
|
730
|
+
if execute:
|
|
731
|
+
args.append("--execute")
|
|
732
|
+
if generate_rollback:
|
|
733
|
+
args.append("--generate-rollback")
|
|
734
|
+
result = subprocess.run(args)
|
|
735
|
+
raise typer.Exit(result.returncode)
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
@app.command("enrich")
|
|
739
|
+
def enrich(
|
|
740
|
+
batch_size: int = typer.Option(50, "--batch-size", "-b", help="Chunks per batch"),
|
|
741
|
+
max_chunks: int = typer.Option(0, "--max", "-m", help="Max chunks (0=unlimited)"),
|
|
742
|
+
no_context: bool = typer.Option(False, "--no-context", help="Skip surrounding context"),
|
|
743
|
+
stats_only: bool = typer.Option(False, "--stats", help="Show progress and exit"),
|
|
744
|
+
) -> None:
|
|
745
|
+
"""Enrich chunks with LLM-generated metadata (summary, tags, importance, intent)."""
|
|
746
|
+
try:
|
|
747
|
+
from ..pipeline.enrichment import DEFAULT_DB_PATH, run_enrichment
|
|
748
|
+
from ..vector_store import VectorStore
|
|
749
|
+
|
|
750
|
+
if stats_only:
|
|
751
|
+
store = VectorStore(DEFAULT_DB_PATH)
|
|
752
|
+
try:
|
|
753
|
+
s = store.get_enrichment_stats()
|
|
754
|
+
console.print(f"[bold]Total:[/] {s['total_chunks']}")
|
|
755
|
+
console.print(f"[bold]Enriched:[/] {s['enriched']} ({s['percent']}%)")
|
|
756
|
+
console.print(f"[bold]Remaining:[/] {s['remaining']}")
|
|
757
|
+
if s["by_intent"]:
|
|
758
|
+
console.print(f"[bold]Intent distribution:[/] {s['by_intent']}")
|
|
759
|
+
finally:
|
|
760
|
+
store.close()
|
|
761
|
+
else:
|
|
762
|
+
run_enrichment(
|
|
763
|
+
batch_size=batch_size,
|
|
764
|
+
max_chunks=max_chunks,
|
|
765
|
+
with_context=not no_context,
|
|
766
|
+
)
|
|
767
|
+
except Exception as e:
|
|
768
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
769
|
+
raise typer.Exit(1)
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
@app.command("enrich-sessions")
|
|
773
|
+
def enrich_sessions(
|
|
774
|
+
project: str = typer.Option(None, "--project", "-p", help="Only enrich sessions from this project"),
|
|
775
|
+
since: str = typer.Option(None, "--since", "-s", help="Only sessions after this date (ISO 8601)"),
|
|
776
|
+
max_sessions: int = typer.Option(0, "--max", "-m", help="Max sessions to process (0=unlimited)"),
|
|
777
|
+
stats_only: bool = typer.Option(False, "--stats", help="Show enrichment stats and exit"),
|
|
778
|
+
) -> None:
|
|
779
|
+
"""Enrich sessions with LLM-generated analysis (summary, decisions, corrections, learnings)."""
|
|
780
|
+
try:
|
|
781
|
+
from ..paths import DEFAULT_DB_PATH
|
|
782
|
+
from ..pipeline.enrichment import ENRICH_BACKEND, call_llm
|
|
783
|
+
from ..pipeline.session_enrichment import (
|
|
784
|
+
enrich_session,
|
|
785
|
+
list_sessions_for_enrichment,
|
|
786
|
+
)
|
|
787
|
+
from ..vector_store import VectorStore
|
|
788
|
+
|
|
789
|
+
store = VectorStore(DEFAULT_DB_PATH)
|
|
790
|
+
|
|
791
|
+
if stats_only:
|
|
792
|
+
try:
|
|
793
|
+
stats = store.get_session_enrichment_stats()
|
|
794
|
+
console.print(f"[bold]Enriched sessions:[/] {stats['total_enriched_sessions']}")
|
|
795
|
+
if stats["by_outcome"]:
|
|
796
|
+
console.print(f"[bold]By outcome:[/] {stats['by_outcome']}")
|
|
797
|
+
if stats["by_intent"]:
|
|
798
|
+
console.print(f"[bold]By intent:[/] {stats['by_intent']}")
|
|
799
|
+
if stats["avg_quality_score"]:
|
|
800
|
+
console.print(f"[bold]Avg quality:[/] {stats['avg_quality_score']}/10")
|
|
801
|
+
finally:
|
|
802
|
+
store.close()
|
|
803
|
+
return
|
|
804
|
+
|
|
805
|
+
sessions = list_sessions_for_enrichment(store, project=project, since=since)
|
|
806
|
+
if max_sessions > 0:
|
|
807
|
+
sessions = sessions[:max_sessions]
|
|
808
|
+
|
|
809
|
+
if not sessions:
|
|
810
|
+
console.print("[yellow]No sessions to enrich.[/]")
|
|
811
|
+
store.close()
|
|
812
|
+
return
|
|
813
|
+
|
|
814
|
+
console.print(f"[bold]Found {len(sessions)} sessions to enrich[/] (backend: {ENRICH_BACKEND})")
|
|
815
|
+
|
|
816
|
+
enriched = 0
|
|
817
|
+
failed = 0
|
|
818
|
+
|
|
819
|
+
with Progress(
|
|
820
|
+
SpinnerColumn(),
|
|
821
|
+
TextColumn("[progress.description]{task.description}"),
|
|
822
|
+
BarColumn(),
|
|
823
|
+
MofNCompleteColumn(),
|
|
824
|
+
TimeElapsedColumn(),
|
|
825
|
+
console=console,
|
|
826
|
+
) as progress:
|
|
827
|
+
task = progress.add_task("Enriching sessions...", total=len(sessions))
|
|
828
|
+
|
|
829
|
+
for i, (session_id, proj, chunk_count) in enumerate(sessions):
|
|
830
|
+
progress.update(task, description=f"Session {session_id[:8]}... ({chunk_count} chunks)")
|
|
831
|
+
|
|
832
|
+
try:
|
|
833
|
+
result = enrich_session(
|
|
834
|
+
store=store,
|
|
835
|
+
session_id=session_id,
|
|
836
|
+
call_llm_fn=call_llm,
|
|
837
|
+
project=proj or project,
|
|
838
|
+
model_name=ENRICH_BACKEND,
|
|
839
|
+
)
|
|
840
|
+
if result:
|
|
841
|
+
enriched += 1
|
|
842
|
+
else:
|
|
843
|
+
failed += 1
|
|
844
|
+
except Exception as e:
|
|
845
|
+
console.print(f" [red]Error on {session_id[:8]}: {e}[/]")
|
|
846
|
+
failed += 1
|
|
847
|
+
|
|
848
|
+
progress.advance(task)
|
|
849
|
+
|
|
850
|
+
store.close()
|
|
851
|
+
|
|
852
|
+
console.print(f"\n[bold green]Done![/] Enriched: {enriched}, Failed: {failed}")
|
|
853
|
+
|
|
854
|
+
except Exception as e:
|
|
855
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
856
|
+
raise typer.Exit(1)
|
|
857
|
+
|
|
858
|
+
|
|
859
|
+
@app.command("git-overlay")
|
|
860
|
+
def git_overlay(
|
|
861
|
+
project: str = typer.Option(None, "--project", "-p", help="Only process specific project slug"),
|
|
862
|
+
force: bool = typer.Option(False, "--force", "-f", help="Re-process sessions that already have context"),
|
|
863
|
+
max_sessions: int = typer.Option(0, "--max", "-m", help="Max sessions to process (0=all)"),
|
|
864
|
+
stats_only: bool = typer.Option(False, "--stats", help="Show stats and exit"),
|
|
865
|
+
file_timeline: str = typer.Option(None, "--file", help="Show timeline for a specific file"),
|
|
866
|
+
) -> None:
|
|
867
|
+
"""Build git overlay: cross-reference sessions with git history (Phase 8b)."""
|
|
868
|
+
from ..vector_store import VectorStore
|
|
869
|
+
|
|
870
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
871
|
+
store = VectorStore(db_path)
|
|
872
|
+
|
|
873
|
+
try:
|
|
874
|
+
if file_timeline:
|
|
875
|
+
results = store.get_file_timeline(file_timeline, project=project)
|
|
876
|
+
if not results:
|
|
877
|
+
console.print(f"[dim]No interactions found for '{file_timeline}'[/]")
|
|
878
|
+
return
|
|
879
|
+
|
|
880
|
+
table = Table(title=f"File Timeline: {file_timeline}")
|
|
881
|
+
table.add_column("Timestamp", style="cyan")
|
|
882
|
+
table.add_column("Action", style="yellow")
|
|
883
|
+
table.add_column("Project", style="green")
|
|
884
|
+
table.add_column("Branch", style="magenta")
|
|
885
|
+
table.add_column("PR", style="blue")
|
|
886
|
+
table.add_column("Session", style="dim")
|
|
887
|
+
|
|
888
|
+
for r in results:
|
|
889
|
+
ts = (r["timestamp"] or "")[:19]
|
|
890
|
+
table.add_row(
|
|
891
|
+
ts,
|
|
892
|
+
r["action"],
|
|
893
|
+
r["project"] or "",
|
|
894
|
+
r["branch"] or "",
|
|
895
|
+
f"#{r['pr_number']}" if r["pr_number"] else "",
|
|
896
|
+
(r["session_id"] or "")[:8],
|
|
897
|
+
)
|
|
898
|
+
console.print(table)
|
|
899
|
+
return
|
|
900
|
+
|
|
901
|
+
if stats_only:
|
|
902
|
+
s = store.get_git_overlay_stats()
|
|
903
|
+
console.print(f"[bold]Sessions with context:[/] {s['sessions_with_context']}")
|
|
904
|
+
console.print(f"[bold]File interactions:[/] {s['file_interactions']}")
|
|
905
|
+
console.print(f"[bold]Unique files tracked:[/] {s['unique_files']}")
|
|
906
|
+
return
|
|
907
|
+
|
|
908
|
+
import logging
|
|
909
|
+
|
|
910
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
911
|
+
|
|
912
|
+
from ..pipeline.git_overlay import run_git_overlay as _run
|
|
913
|
+
|
|
914
|
+
console.print("[bold]Running git overlay...[/]")
|
|
915
|
+
result = _run(
|
|
916
|
+
vector_store=store,
|
|
917
|
+
project=project,
|
|
918
|
+
force=force,
|
|
919
|
+
max_sessions=max_sessions,
|
|
920
|
+
)
|
|
921
|
+
console.print(
|
|
922
|
+
f"[green]Done![/] Sessions: {result['sessions_processed']}, "
|
|
923
|
+
f"File interactions: {result['file_interactions_added']}"
|
|
924
|
+
)
|
|
925
|
+
except Exception as e:
|
|
926
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
927
|
+
raise typer.Exit(1)
|
|
928
|
+
finally:
|
|
929
|
+
store.close()
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
@app.command("group-operations")
|
|
933
|
+
def group_operations(
|
|
934
|
+
project: str = typer.Option(None, "--project", "-p", help="Only process specific project name"),
|
|
935
|
+
force: bool = typer.Option(False, "--force", "-f", help="Re-process sessions with existing operations"),
|
|
936
|
+
max_sessions: int = typer.Option(0, "--max", "-m", help="Max sessions to process (0=all)"),
|
|
937
|
+
stats_only: bool = typer.Option(False, "--stats", help="Show stats and exit"),
|
|
938
|
+
session: str = typer.Option(None, "--session", "-s", help="Show operations for a specific session"),
|
|
939
|
+
) -> None:
|
|
940
|
+
"""Group chunks into logical operations (Phase 8a)."""
|
|
941
|
+
from ..vector_store import VectorStore
|
|
942
|
+
|
|
943
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
944
|
+
store = VectorStore(db_path)
|
|
945
|
+
|
|
946
|
+
try:
|
|
947
|
+
if session:
|
|
948
|
+
ops = store.get_session_operations(session)
|
|
949
|
+
if not ops:
|
|
950
|
+
console.print(f"[dim]No operations for session '{session[:8]}...'[/]")
|
|
951
|
+
return
|
|
952
|
+
|
|
953
|
+
table = Table(title=f"Operations: {session[:8]}...")
|
|
954
|
+
table.add_column("Type", style="cyan")
|
|
955
|
+
table.add_column("Summary", style="white")
|
|
956
|
+
table.add_column("Steps", style="yellow")
|
|
957
|
+
table.add_column("Outcome", style="green")
|
|
958
|
+
table.add_column("Started", style="dim")
|
|
959
|
+
|
|
960
|
+
for op in ops:
|
|
961
|
+
outcome_style = {
|
|
962
|
+
"success": "[green]success[/]",
|
|
963
|
+
"failure": "[red]failure[/]",
|
|
964
|
+
}.get(
|
|
965
|
+
op["outcome"],
|
|
966
|
+
f"[dim]{op['outcome']}[/]",
|
|
967
|
+
)
|
|
968
|
+
ts = (op["started_at"] or "")[:19]
|
|
969
|
+
table.add_row(
|
|
970
|
+
op["operation_type"],
|
|
971
|
+
op["summary"],
|
|
972
|
+
str(op["step_count"]),
|
|
973
|
+
outcome_style,
|
|
974
|
+
ts,
|
|
975
|
+
)
|
|
976
|
+
console.print(table)
|
|
977
|
+
return
|
|
978
|
+
|
|
979
|
+
if stats_only:
|
|
980
|
+
s = store.get_operations_stats()
|
|
981
|
+
console.print(f"[bold]Total operations:[/] {s['total_operations']}")
|
|
982
|
+
console.print(f"[bold]Sessions with operations:[/] {s['sessions_with_operations']}")
|
|
983
|
+
if s["by_type"]:
|
|
984
|
+
table = Table(title="By Type")
|
|
985
|
+
table.add_column("Type", style="cyan")
|
|
986
|
+
table.add_column("Count", style="yellow")
|
|
987
|
+
for t, c in s["by_type"].items():
|
|
988
|
+
table.add_row(t, str(c))
|
|
989
|
+
console.print(table)
|
|
990
|
+
return
|
|
991
|
+
|
|
992
|
+
import logging
|
|
993
|
+
|
|
994
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
995
|
+
|
|
996
|
+
from ..pipeline.operation_grouping import (
|
|
997
|
+
run_operation_grouping as _run,
|
|
998
|
+
)
|
|
999
|
+
|
|
1000
|
+
console.print("[bold]Grouping operations...[/]")
|
|
1001
|
+
result = _run(
|
|
1002
|
+
vector_store=store,
|
|
1003
|
+
project=project,
|
|
1004
|
+
force=force,
|
|
1005
|
+
max_sessions=max_sessions,
|
|
1006
|
+
)
|
|
1007
|
+
console.print(
|
|
1008
|
+
f"[green]Done![/] Sessions: {result['sessions_processed']}, Operations: {result['operations_added']}"
|
|
1009
|
+
)
|
|
1010
|
+
except Exception as e:
|
|
1011
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1012
|
+
raise typer.Exit(1)
|
|
1013
|
+
finally:
|
|
1014
|
+
store.close()
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
@app.command("topic-chains")
|
|
1018
|
+
def topic_chains(
|
|
1019
|
+
project: str = typer.Option(None, "--project", "-p", help="Filter by project name"),
|
|
1020
|
+
force: bool = typer.Option(False, "--force", "-f", help="Clear and rebuild chains"),
|
|
1021
|
+
stats_only: bool = typer.Option(False, "--stats", help="Show stats and exit"),
|
|
1022
|
+
file_query: str = typer.Option(None, "--file", help="Show chains for a specific file"),
|
|
1023
|
+
regression: str = typer.Option(None, "--regression", help="Show regression analysis for a file"),
|
|
1024
|
+
) -> None:
|
|
1025
|
+
"""Build topic chains + regression detection (Phase 8d)."""
|
|
1026
|
+
from ..vector_store import VectorStore
|
|
1027
|
+
|
|
1028
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
1029
|
+
store = VectorStore(db_path)
|
|
1030
|
+
|
|
1031
|
+
try:
|
|
1032
|
+
if regression:
|
|
1033
|
+
result = store.get_file_regression(regression, project=project)
|
|
1034
|
+
if not result["timeline"]:
|
|
1035
|
+
console.print(f"[dim]No interactions for '{regression}'[/]")
|
|
1036
|
+
return
|
|
1037
|
+
|
|
1038
|
+
console.print(f"[bold]Regression: {regression}[/]")
|
|
1039
|
+
console.print(f"Timeline entries: {len(result['timeline'])}")
|
|
1040
|
+
|
|
1041
|
+
if result["last_success"]:
|
|
1042
|
+
ls = result["last_success"]
|
|
1043
|
+
console.print(
|
|
1044
|
+
f"[green]Last success:[/]"
|
|
1045
|
+
f" {ls['timestamp']}"
|
|
1046
|
+
f" (session {ls['session_id'][:8]},"
|
|
1047
|
+
f" branch {ls.get('branch', '?')})"
|
|
1048
|
+
)
|
|
1049
|
+
else:
|
|
1050
|
+
console.print("[yellow]No successful operations found[/]")
|
|
1051
|
+
|
|
1052
|
+
if result["changes_after"]:
|
|
1053
|
+
table = Table(title="Changes After Last Success")
|
|
1054
|
+
table.add_column("Timestamp", style="cyan")
|
|
1055
|
+
table.add_column("Action", style="yellow")
|
|
1056
|
+
table.add_column("Branch", style="magenta")
|
|
1057
|
+
table.add_column("Session", style="dim")
|
|
1058
|
+
for c in result["changes_after"][:20]:
|
|
1059
|
+
ts = (c["timestamp"] or "")[:19]
|
|
1060
|
+
table.add_row(
|
|
1061
|
+
ts,
|
|
1062
|
+
c["action"] or "?",
|
|
1063
|
+
c.get("branch") or "",
|
|
1064
|
+
(c["session_id"] or "")[:8],
|
|
1065
|
+
)
|
|
1066
|
+
console.print(table)
|
|
1067
|
+
return
|
|
1068
|
+
|
|
1069
|
+
if file_query:
|
|
1070
|
+
chains = store.get_file_chains(file_query)
|
|
1071
|
+
if not chains:
|
|
1072
|
+
console.print(f"[dim]No chains for '{file_query}'[/]")
|
|
1073
|
+
return
|
|
1074
|
+
|
|
1075
|
+
table = Table(title=f"Topic Chains: {file_query}")
|
|
1076
|
+
table.add_column("Session A", style="cyan")
|
|
1077
|
+
table.add_column("Session B", style="green")
|
|
1078
|
+
table.add_column("Delta (hrs)", style="yellow")
|
|
1079
|
+
table.add_column("Shared", style="magenta")
|
|
1080
|
+
table.add_column("Branch A", style="dim")
|
|
1081
|
+
table.add_column("Branch B", style="dim")
|
|
1082
|
+
for c in chains:
|
|
1083
|
+
delta = f"{c['time_delta_hours']:.1f}" if c["time_delta_hours"] is not None else "?"
|
|
1084
|
+
table.add_row(
|
|
1085
|
+
(c["session_a"] or "")[:8],
|
|
1086
|
+
(c["session_b"] or "")[:8],
|
|
1087
|
+
delta,
|
|
1088
|
+
str(c["shared_actions"]),
|
|
1089
|
+
c.get("branch_a") or "",
|
|
1090
|
+
c.get("branch_b") or "",
|
|
1091
|
+
)
|
|
1092
|
+
console.print(table)
|
|
1093
|
+
return
|
|
1094
|
+
|
|
1095
|
+
if stats_only:
|
|
1096
|
+
s = store.get_topic_chain_stats()
|
|
1097
|
+
console.print(f"[bold]Total chains:[/] {s['total_chains']}")
|
|
1098
|
+
console.print(f"[bold]Unique files:[/] {s['unique_files']}")
|
|
1099
|
+
return
|
|
1100
|
+
|
|
1101
|
+
import logging
|
|
1102
|
+
|
|
1103
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
1104
|
+
|
|
1105
|
+
from ..pipeline.temporal_chains import (
|
|
1106
|
+
run_temporal_chains as _run,
|
|
1107
|
+
)
|
|
1108
|
+
|
|
1109
|
+
console.print("[bold]Building topic chains...[/]")
|
|
1110
|
+
result = _run(
|
|
1111
|
+
vector_store=store,
|
|
1112
|
+
project=project,
|
|
1113
|
+
force=force,
|
|
1114
|
+
)
|
|
1115
|
+
console.print(f"[green]Done![/] Files: {result['files_analyzed']}, Chains: {result['chains_created']}")
|
|
1116
|
+
except Exception as e:
|
|
1117
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1118
|
+
raise typer.Exit(1)
|
|
1119
|
+
finally:
|
|
1120
|
+
store.close()
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
@app.command("plan-linking")
|
|
1124
|
+
def plan_linking(
|
|
1125
|
+
project: str = typer.Option(None, "--project", "-p", help="Filter by project name"),
|
|
1126
|
+
force: bool = typer.Option(False, "--force", "-f", help="Clear and rebuild plan links"),
|
|
1127
|
+
stats_only: bool = typer.Option(False, "--stats", help="Show stats and exit"),
|
|
1128
|
+
plan_query: str = typer.Option(None, "--plan", help="Show sessions for a specific plan"),
|
|
1129
|
+
session_query: str = typer.Option(None, "--session", help="Show plan info for a session ID"),
|
|
1130
|
+
repo_root: str = typer.Option(None, "--repo", help="Repo root path (auto-detected)"),
|
|
1131
|
+
) -> None:
|
|
1132
|
+
"""Link sessions to active plans (Phase 8c)."""
|
|
1133
|
+
from ..vector_store import VectorStore
|
|
1134
|
+
|
|
1135
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
1136
|
+
store = VectorStore(db_path)
|
|
1137
|
+
|
|
1138
|
+
try:
|
|
1139
|
+
if session_query:
|
|
1140
|
+
ctx = store.get_session_context(session_query)
|
|
1141
|
+
if not ctx:
|
|
1142
|
+
# Try prefix match
|
|
1143
|
+
cursor = store.conn.cursor()
|
|
1144
|
+
rows = list(
|
|
1145
|
+
cursor.execute(
|
|
1146
|
+
"SELECT session_id FROM session_context WHERE session_id LIKE ?",
|
|
1147
|
+
(f"{session_query}%",),
|
|
1148
|
+
)
|
|
1149
|
+
)
|
|
1150
|
+
if len(rows) == 1:
|
|
1151
|
+
ctx = store.get_session_context(rows[0][0])
|
|
1152
|
+
elif len(rows) > 1:
|
|
1153
|
+
console.print(f"[yellow]Multiple sessions match '{session_query}':[/]")
|
|
1154
|
+
for r in rows[:10]:
|
|
1155
|
+
console.print(f" {r[0]}")
|
|
1156
|
+
return
|
|
1157
|
+
if not ctx:
|
|
1158
|
+
console.print(f"[dim]No context for session '{session_query[:8]}'[/]")
|
|
1159
|
+
return
|
|
1160
|
+
console.print(f"[bold]Session:[/] {ctx['session_id'][:8]}")
|
|
1161
|
+
console.print(f" Branch: {ctx.get('branch') or '?'}")
|
|
1162
|
+
console.print(f" PR: #{ctx.get('pr_number') or '?'}")
|
|
1163
|
+
console.print(f" Plan: {ctx.get('plan_name') or '(none)'}")
|
|
1164
|
+
console.print(f" Phase: {ctx.get('plan_phase') or '(none)'}")
|
|
1165
|
+
console.print(f" Story: {ctx.get('story_id') or '(none)'}")
|
|
1166
|
+
return
|
|
1167
|
+
|
|
1168
|
+
if plan_query:
|
|
1169
|
+
sessions = store.get_sessions_by_plan(plan_name=plan_query, project=project)
|
|
1170
|
+
if not sessions:
|
|
1171
|
+
console.print(f"[dim]No sessions for plan '{plan_query}'[/]")
|
|
1172
|
+
return
|
|
1173
|
+
|
|
1174
|
+
table = Table(title=f"Sessions: {plan_query}")
|
|
1175
|
+
table.add_column("Session", style="cyan")
|
|
1176
|
+
table.add_column("Branch", style="green")
|
|
1177
|
+
table.add_column("PR", style="yellow")
|
|
1178
|
+
table.add_column("Phase", style="magenta")
|
|
1179
|
+
table.add_column("Started", style="dim")
|
|
1180
|
+
for s in sessions:
|
|
1181
|
+
table.add_row(
|
|
1182
|
+
(s["session_id"] or "")[:8],
|
|
1183
|
+
s.get("branch") or "",
|
|
1184
|
+
f"#{s['pr_number']}" if s.get("pr_number") else "",
|
|
1185
|
+
s.get("plan_phase") or "",
|
|
1186
|
+
(s.get("started_at") or "")[:19],
|
|
1187
|
+
)
|
|
1188
|
+
console.print(table)
|
|
1189
|
+
return
|
|
1190
|
+
|
|
1191
|
+
if stats_only:
|
|
1192
|
+
s = store.get_plan_linking_stats()
|
|
1193
|
+
console.print(f"[bold]Total sessions:[/] {s['total_sessions']}")
|
|
1194
|
+
console.print(f"[bold]Linked:[/] {s['linked_sessions']}")
|
|
1195
|
+
console.print(f"[bold]Unlinked:[/] {s['unlinked_sessions']}")
|
|
1196
|
+
if s["plans"]:
|
|
1197
|
+
table = Table(title="Plans")
|
|
1198
|
+
table.add_column("Plan", style="cyan")
|
|
1199
|
+
table.add_column("Sessions", style="green")
|
|
1200
|
+
for name, count in s["plans"].items():
|
|
1201
|
+
table.add_row(name, str(count))
|
|
1202
|
+
console.print(table)
|
|
1203
|
+
return
|
|
1204
|
+
|
|
1205
|
+
import logging
|
|
1206
|
+
|
|
1207
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
1208
|
+
|
|
1209
|
+
from ..pipeline.plan_linking import (
|
|
1210
|
+
run_plan_linking as _run,
|
|
1211
|
+
)
|
|
1212
|
+
|
|
1213
|
+
console.print("[bold]Linking sessions to plans...[/]")
|
|
1214
|
+
result = _run(
|
|
1215
|
+
vector_store=store,
|
|
1216
|
+
repo_root=repo_root,
|
|
1217
|
+
project=project,
|
|
1218
|
+
force=force,
|
|
1219
|
+
)
|
|
1220
|
+
console.print(f"[green]Done![/] Checked: {result['sessions_checked']}, Linked: {result['sessions_linked']}")
|
|
1221
|
+
except Exception as e:
|
|
1222
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1223
|
+
raise typer.Exit(1)
|
|
1224
|
+
finally:
|
|
1225
|
+
store.close()
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
@app.command("export-obsidian")
|
|
1229
|
+
def export_obsidian(
|
|
1230
|
+
vault_path: str = typer.Option(
|
|
1231
|
+
None, "--vault", "-v", help="Obsidian vault path (default: ~/.brainlayer-brain/BrainLayer)"
|
|
1232
|
+
),
|
|
1233
|
+
project: str = typer.Option(None, "--project", "-p", help="Filter by project name"),
|
|
1234
|
+
force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing notes"),
|
|
1235
|
+
) -> None:
|
|
1236
|
+
"""Export BrainLayer data to Obsidian vault (Phase 9)."""
|
|
1237
|
+
from ..vector_store import VectorStore
|
|
1238
|
+
|
|
1239
|
+
db_path = Path.home() / ".local" / "share" / "brainlayer" / "brainlayer.db"
|
|
1240
|
+
store = VectorStore(db_path)
|
|
1241
|
+
|
|
1242
|
+
try:
|
|
1243
|
+
import logging
|
|
1244
|
+
|
|
1245
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
1246
|
+
|
|
1247
|
+
from ..pipeline.obsidian_export import (
|
|
1248
|
+
export_obsidian as _export,
|
|
1249
|
+
)
|
|
1250
|
+
|
|
1251
|
+
console.print("[bold]Exporting to Obsidian vault...[/]")
|
|
1252
|
+
result = _export(
|
|
1253
|
+
vector_store=store,
|
|
1254
|
+
vault_path=vault_path,
|
|
1255
|
+
project=project,
|
|
1256
|
+
force=force,
|
|
1257
|
+
)
|
|
1258
|
+
console.print(
|
|
1259
|
+
f"[green]Done![/]"
|
|
1260
|
+
f" Sessions: {result['sessions']},"
|
|
1261
|
+
f" Files: {result['files']},"
|
|
1262
|
+
f" Plans: {result['plans']},"
|
|
1263
|
+
f" Dashboards: {result['dashboards']}"
|
|
1264
|
+
)
|
|
1265
|
+
vault = vault_path or str(Path.home() / ".brainlayer-brain" / "BrainLayer")
|
|
1266
|
+
console.print(f"[dim]Vault: {vault}[/]")
|
|
1267
|
+
except Exception as e:
|
|
1268
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1269
|
+
raise typer.Exit(1)
|
|
1270
|
+
finally:
|
|
1271
|
+
store.close()
|
|
1272
|
+
|
|
1273
|
+
|
|
1274
|
+
@app.command("index-fast", hidden=True)
|
|
1275
|
+
def index_fast(
|
|
1276
|
+
source: Path = typer.Argument(
|
|
1277
|
+
Path.home() / ".claude" / "projects", help="Source directory containing JSONL conversations"
|
|
1278
|
+
),
|
|
1279
|
+
project: str = typer.Option(None, "--project", "-p", help="Only index specific project (folder name)"),
|
|
1280
|
+
force: bool = typer.Option(False, "--force", "-f", help="Re-index all files (ignore cache)"),
|
|
1281
|
+
) -> None:
|
|
1282
|
+
"""Index using new fast sqlite-vec backend."""
|
|
1283
|
+
try:
|
|
1284
|
+
import time
|
|
1285
|
+
|
|
1286
|
+
from rich.progress import (
|
|
1287
|
+
BarColumn,
|
|
1288
|
+
MofNCompleteColumn,
|
|
1289
|
+
Progress,
|
|
1290
|
+
SpinnerColumn,
|
|
1291
|
+
TaskProgressColumn,
|
|
1292
|
+
TextColumn,
|
|
1293
|
+
TimeElapsedColumn,
|
|
1294
|
+
TimeRemainingColumn,
|
|
1295
|
+
)
|
|
1296
|
+
|
|
1297
|
+
from ..index_new import index_chunks_to_sqlite
|
|
1298
|
+
from ..pipeline.chunk import chunk_content
|
|
1299
|
+
from ..pipeline.classify import classify_content
|
|
1300
|
+
from ..pipeline.extract import parse_jsonl
|
|
1301
|
+
|
|
1302
|
+
if not source.exists():
|
|
1303
|
+
rprint(f"[bold red]Error:[/] Source directory not found: {source}")
|
|
1304
|
+
raise typer.Exit(1)
|
|
1305
|
+
|
|
1306
|
+
# Find JSONL files
|
|
1307
|
+
if project:
|
|
1308
|
+
project_dir = source / project
|
|
1309
|
+
if not project_dir.exists():
|
|
1310
|
+
rprint(f"[bold red]Error:[/] Project directory not found: {project_dir}")
|
|
1311
|
+
raise typer.Exit(1)
|
|
1312
|
+
jsonl_files = list(project_dir.glob("*.jsonl"))
|
|
1313
|
+
if not jsonl_files:
|
|
1314
|
+
rprint(f"[bold red]Error:[/] No JSONL files found in project: {project_dir}")
|
|
1315
|
+
raise typer.Exit(1)
|
|
1316
|
+
else:
|
|
1317
|
+
jsonl_files = list(source.rglob("*.jsonl"))
|
|
1318
|
+
if not jsonl_files:
|
|
1319
|
+
rprint(f"[bold red]Error:[/] No JSONL files found in: {source}")
|
|
1320
|
+
raise typer.Exit(1)
|
|
1321
|
+
|
|
1322
|
+
rprint(f"[bold blue]זיכרון[/] - Fast Indexing: [bold]{len(jsonl_files)}[/] files")
|
|
1323
|
+
|
|
1324
|
+
total_chunks = 0
|
|
1325
|
+
start_time = time.time()
|
|
1326
|
+
|
|
1327
|
+
with Progress(
|
|
1328
|
+
SpinnerColumn(),
|
|
1329
|
+
TextColumn("[progress.description]{task.description}"),
|
|
1330
|
+
BarColumn(),
|
|
1331
|
+
MofNCompleteColumn(),
|
|
1332
|
+
TaskProgressColumn(),
|
|
1333
|
+
TimeElapsedColumn(),
|
|
1334
|
+
TimeRemainingColumn(),
|
|
1335
|
+
console=console,
|
|
1336
|
+
) as progress:
|
|
1337
|
+
task = progress.add_task("Processing files...", total=len(jsonl_files))
|
|
1338
|
+
|
|
1339
|
+
for i, jsonl_file in enumerate(jsonl_files):
|
|
1340
|
+
raw_proj = jsonl_file.parent.name if jsonl_file.parent != source else None
|
|
1341
|
+
proj_name = _normalize_project_name(raw_proj) if raw_proj else None
|
|
1342
|
+
|
|
1343
|
+
# Parse, classify, and chunk each entry
|
|
1344
|
+
all_chunks = []
|
|
1345
|
+
for entry in parse_jsonl(jsonl_file):
|
|
1346
|
+
classified = classify_content(entry)
|
|
1347
|
+
if classified is not None: # Skip noise entries
|
|
1348
|
+
chunks = chunk_content(classified)
|
|
1349
|
+
all_chunks.extend(chunks)
|
|
1350
|
+
|
|
1351
|
+
if all_chunks:
|
|
1352
|
+
# Index with progress callback
|
|
1353
|
+
def progress_callback(embedded_count, total_embed):
|
|
1354
|
+
pass # Could update sub-progress here
|
|
1355
|
+
|
|
1356
|
+
indexed = index_chunks_to_sqlite(
|
|
1357
|
+
all_chunks,
|
|
1358
|
+
source_file=str(jsonl_file),
|
|
1359
|
+
project=proj_name,
|
|
1360
|
+
on_progress=progress_callback,
|
|
1361
|
+
)
|
|
1362
|
+
total_chunks += indexed
|
|
1363
|
+
|
|
1364
|
+
# Update progress
|
|
1365
|
+
elapsed = time.time() - start_time
|
|
1366
|
+
files_done = i + 1
|
|
1367
|
+
rate_per_min = (files_done / elapsed) * 60 if elapsed > 0 else 0
|
|
1368
|
+
chunks_per_min = (total_chunks / elapsed) * 60 if elapsed > 0 else 0
|
|
1369
|
+
|
|
1370
|
+
progress.update(
|
|
1371
|
+
task,
|
|
1372
|
+
completed=files_done,
|
|
1373
|
+
description=f"[cyan]{jsonl_file.name[:30]}[/] • {total_chunks:,} chunks • {rate_per_min:.1f} f/m • {chunks_per_min:.0f} c/m",
|
|
1374
|
+
)
|
|
1375
|
+
|
|
1376
|
+
elapsed_total = time.time() - start_time
|
|
1377
|
+
rprint(
|
|
1378
|
+
f"\n[bold green]✓[/] Indexed [bold]{total_chunks:,}[/] chunks from [bold]{len(jsonl_files):,}[/] files in [bold]{elapsed_total / 60:.1f}[/] minutes"
|
|
1379
|
+
)
|
|
1380
|
+
|
|
1381
|
+
# Sync enrichment stats to Supabase (best-effort)
|
|
1382
|
+
try:
|
|
1383
|
+
from ..pipeline.enrichment import DEFAULT_DB_PATH, _sync_stats_to_supabase
|
|
1384
|
+
from ..vector_store import VectorStore
|
|
1385
|
+
|
|
1386
|
+
store = VectorStore(DEFAULT_DB_PATH)
|
|
1387
|
+
_sync_stats_to_supabase(store)
|
|
1388
|
+
store.close()
|
|
1389
|
+
rprint("[dim]Synced enrichment stats to Supabase[/]")
|
|
1390
|
+
except Exception:
|
|
1391
|
+
pass
|
|
1392
|
+
|
|
1393
|
+
except typer.Exit:
|
|
1394
|
+
raise
|
|
1395
|
+
except Exception as e:
|
|
1396
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1397
|
+
raise typer.Exit(1)
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
@app.command("analyze-semantic")
|
|
1401
|
+
def analyze_semantic(
|
|
1402
|
+
whatsapp_limit: int = typer.Option(5000, "--whatsapp-limit", "-w", help="Number of WhatsApp messages to analyze"),
|
|
1403
|
+
claude_export: Path = typer.Option(
|
|
1404
|
+
None, "--claude-export", "-c", help="Path to Claude chat export JSON (optional)"
|
|
1405
|
+
),
|
|
1406
|
+
output_dir: Path = typer.Option(
|
|
1407
|
+
Path.home() / ".local/share/brainlayer/style",
|
|
1408
|
+
"--output",
|
|
1409
|
+
"-o",
|
|
1410
|
+
help="Output directory for semantic style rules",
|
|
1411
|
+
),
|
|
1412
|
+
min_cluster: int = typer.Option(10, "--min-cluster", "-m", help="Minimum messages per topic cluster"),
|
|
1413
|
+
) -> None:
|
|
1414
|
+
"""Analyze communication style by topic using semantic clustering.
|
|
1415
|
+
|
|
1416
|
+
Clusters your messages by topic (technical, casual, professional, etc.)
|
|
1417
|
+
and analyzes writing style patterns within each context. Generates
|
|
1418
|
+
context-aware rules for better cover letters and outreach.
|
|
1419
|
+
|
|
1420
|
+
Requires: sentence-transformers, scikit-learn
|
|
1421
|
+
"""
|
|
1422
|
+
try:
|
|
1423
|
+
from ..pipeline.analyze_communication import CommunicationAnalyzer
|
|
1424
|
+
from ..pipeline.extract_whatsapp import extract_whatsapp_messages
|
|
1425
|
+
|
|
1426
|
+
rprint("[bold blue]זיכרון[/] - Semantic Style Analysis\n")
|
|
1427
|
+
|
|
1428
|
+
analyzer = CommunicationAnalyzer()
|
|
1429
|
+
|
|
1430
|
+
# Extract WhatsApp messages
|
|
1431
|
+
with console.status("[bold green]Extracting WhatsApp messages..."):
|
|
1432
|
+
try:
|
|
1433
|
+
messages = list(
|
|
1434
|
+
extract_whatsapp_messages(
|
|
1435
|
+
limit=whatsapp_limit,
|
|
1436
|
+
only_from_me=True, # Only user's messages
|
|
1437
|
+
exclude_groups=True,
|
|
1438
|
+
)
|
|
1439
|
+
)
|
|
1440
|
+
analyzer.add_whatsapp_messages(messages)
|
|
1441
|
+
rprint(f"[green]✓[/] Extracted {len(messages)} WhatsApp messages")
|
|
1442
|
+
except FileNotFoundError as e:
|
|
1443
|
+
rprint("[yellow]Warning:[/] WhatsApp database not found")
|
|
1444
|
+
rprint(f"[dim] {e}[/]")
|
|
1445
|
+
except Exception as e:
|
|
1446
|
+
rprint(f"[yellow]Warning:[/] Failed to extract WhatsApp: {e}")
|
|
1447
|
+
|
|
1448
|
+
# Extract Claude chats if provided
|
|
1449
|
+
if claude_export and claude_export.exists():
|
|
1450
|
+
with console.status("[bold green]Extracting Claude conversations..."):
|
|
1451
|
+
try:
|
|
1452
|
+
import json
|
|
1453
|
+
|
|
1454
|
+
with open(claude_export) as f:
|
|
1455
|
+
data = json.load(f)
|
|
1456
|
+
conversations = data if isinstance(data, list) else data.get("conversations", [])
|
|
1457
|
+
analyzer.add_claude_conversations(conversations)
|
|
1458
|
+
rprint(f"[green]✓[/] Extracted {len(conversations)} Claude conversations")
|
|
1459
|
+
except Exception as e:
|
|
1460
|
+
rprint(f"[yellow]Warning:[/] Failed to extract Claude chats: {e}")
|
|
1461
|
+
else:
|
|
1462
|
+
rprint("[dim]No Claude export provided (use --claude-export)[/]")
|
|
1463
|
+
|
|
1464
|
+
# Check if we have enough data
|
|
1465
|
+
if len(analyzer.user_messages) < min_cluster * 2:
|
|
1466
|
+
rprint(f"[bold red]Error:[/] Need at least {min_cluster * 2} messages, have {len(analyzer.user_messages)}")
|
|
1467
|
+
raise typer.Exit(1)
|
|
1468
|
+
|
|
1469
|
+
rprint(f"\n[bold]Analyzing {len(analyzer.user_messages)} messages...[/]\n")
|
|
1470
|
+
|
|
1471
|
+
# Run semantic analysis
|
|
1472
|
+
rules_path = analyzer.generate_semantic_rules(
|
|
1473
|
+
output_dir=output_dir,
|
|
1474
|
+
min_cluster_size=min_cluster,
|
|
1475
|
+
)
|
|
1476
|
+
|
|
1477
|
+
if rules_path is None:
|
|
1478
|
+
rprint("[bold red]Error:[/] Semantic analysis failed")
|
|
1479
|
+
rprint("[dim]Install dependencies: pip install sentence-transformers scikit-learn[/]")
|
|
1480
|
+
raise typer.Exit(1)
|
|
1481
|
+
|
|
1482
|
+
rprint("\n[bold green]✓[/] Semantic style rules generated!")
|
|
1483
|
+
rprint(f" Markdown: [cyan]{rules_path}[/]")
|
|
1484
|
+
rprint(" JSON: [cyan]semantic-style-data.json[/]")
|
|
1485
|
+
|
|
1486
|
+
# Show preview
|
|
1487
|
+
rprint("\n[bold]Preview:[/]\n")
|
|
1488
|
+
preview = rules_path.read_text()
|
|
1489
|
+
for line in preview.split("\n")[:30]:
|
|
1490
|
+
rprint(f"[dim]{line}[/]")
|
|
1491
|
+
if len(preview.split("\n")) > 30:
|
|
1492
|
+
rprint(f"[dim]... ({len(preview.split(chr(10))) - 30} more lines)[/]")
|
|
1493
|
+
|
|
1494
|
+
except typer.Exit:
|
|
1495
|
+
raise
|
|
1496
|
+
except Exception as e:
|
|
1497
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1498
|
+
import traceback
|
|
1499
|
+
|
|
1500
|
+
traceback.print_exc()
|
|
1501
|
+
raise typer.Exit(1)
|
|
1502
|
+
|
|
1503
|
+
|
|
1504
|
+
@app.command("brain-export")
|
|
1505
|
+
def brain_export(
|
|
1506
|
+
output: str = typer.Option(None, "--output", "-o", help="Output directory (default: ~/.brainlayer-brain)"),
|
|
1507
|
+
project: str = typer.Option(None, "--project", "-p", help="Only include sessions from this project"),
|
|
1508
|
+
) -> None:
|
|
1509
|
+
"""Generate brain graph (graph.json) for 3D visualization.
|
|
1510
|
+
|
|
1511
|
+
Aggregates sessions → computes similarity → runs Leiden communities → UMAP 3D layout.
|
|
1512
|
+
Requires: pip install -e '.[brain]' (igraph, leidenalg, umap-learn)
|
|
1513
|
+
"""
|
|
1514
|
+
try:
|
|
1515
|
+
import logging
|
|
1516
|
+
|
|
1517
|
+
from ..pipeline.brain_graph import generate_brain_graph
|
|
1518
|
+
|
|
1519
|
+
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
|
1520
|
+
|
|
1521
|
+
path = generate_brain_graph(output_dir=output, project=project)
|
|
1522
|
+
rprint(f"[bold green]Brain graph exported to {path}[/]")
|
|
1523
|
+
|
|
1524
|
+
import json
|
|
1525
|
+
|
|
1526
|
+
meta_path = path.parent / "metadata.json"
|
|
1527
|
+
if meta_path.exists():
|
|
1528
|
+
with open(meta_path) as f:
|
|
1529
|
+
meta = json.load(f)
|
|
1530
|
+
rprint(f" Nodes: {meta['node_count']}, Edges: {meta['edge_count']}")
|
|
1531
|
+
rprint(f" Communities: {meta['community_counts']}")
|
|
1532
|
+
except ImportError as e:
|
|
1533
|
+
rprint(f"[bold red]Missing dependency:[/] {e}")
|
|
1534
|
+
rprint("[dim]Install brain extras: pip install -e '.[brain]'[/]")
|
|
1535
|
+
raise typer.Exit(1)
|
|
1536
|
+
except Exception as e:
|
|
1537
|
+
rprint(f"[bold red]Error:[/] {e}")
|
|
1538
|
+
import traceback
|
|
1539
|
+
|
|
1540
|
+
traceback.print_exc()
|
|
1541
|
+
raise typer.Exit(1)
|
|
1542
|
+
|
|
1543
|
+
|
|
1544
|
+
if __name__ == "__main__":
|
|
1545
|
+
app()
|