cli-mem 0.3.4__tar.gz → 0.3.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cli_mem-0.3.4 → cli_mem-0.3.6}/PKG-INFO +1 -1
- {cli_mem-0.3.4 → cli_mem-0.3.6}/pyproject.toml +1 -1
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/__init__.py +1 -1
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/capture.py +26 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/cli.py +13 -21
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/models.py +1 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/patterns.py +65 -8
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/storage.py +30 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/variables.py +66 -13
- {cli_mem-0.3.4 → cli_mem-0.3.6}/tests/conftest.py +4 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/tests/test_groups.py +107 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/tests/test_patterns.py +107 -1
- {cli_mem-0.3.4 → cli_mem-0.3.6}/.github/workflows/ci.yml +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/.github/workflows/release.yml +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/.gitignore +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/ARCHITECTURE.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/LICENSE +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/PHILOSOPHY.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/README.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/assets/.gitkeep +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/docs/decisions/001-jsonl-over-sqlite.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/docs/decisions/002-apple-fm-sdk-for-patterns.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/docs/decisions/003-no-daemon.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/docs/decisions/004-per-repo-jsonl.md +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/hooks/mem.zsh +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/install.sh +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/_generable.py +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/groups.py +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/src/mem/search.py +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/tests/test_capture.py +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/tests/test_search.py +0 -0
- {cli_mem-0.3.4 → cli_mem-0.3.6}/tests/test_storage.py +0 -0
|
@@ -70,6 +70,32 @@ def capture_command(raw: str, directory: str, exit_code: int, duration_ms: int)
|
|
|
70
70
|
except Exception:
|
|
71
71
|
pass # Session tracking failure should never block capture
|
|
72
72
|
|
|
73
|
+
# Auto-sync: trigger background pattern extraction every N captures
|
|
74
|
+
try:
|
|
75
|
+
count = storage.increment_sync_counter()
|
|
76
|
+
if count >= storage.SYNC_THRESHOLD:
|
|
77
|
+
storage.reset_sync_counter()
|
|
78
|
+
_spawn_background_sync()
|
|
79
|
+
except Exception:
|
|
80
|
+
pass # Auto-sync failure should never block capture
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _spawn_background_sync() -> None:
|
|
84
|
+
"""Spawn `mem _sync` as a fully detached background process.
|
|
85
|
+
|
|
86
|
+
The subprocess inherits nothing from the parent — no stdout, no stderr,
|
|
87
|
+
no wait. It runs and exits silently.
|
|
88
|
+
"""
|
|
89
|
+
import sys as _sys
|
|
90
|
+
|
|
91
|
+
mem_exe = _sys.executable
|
|
92
|
+
subprocess.Popen(
|
|
93
|
+
[mem_exe, "-m", "mem.cli", "_sync"],
|
|
94
|
+
stdout=subprocess.DEVNULL,
|
|
95
|
+
stderr=subprocess.DEVNULL,
|
|
96
|
+
start_new_session=True,
|
|
97
|
+
)
|
|
98
|
+
|
|
73
99
|
|
|
74
100
|
class SessionTracker:
|
|
75
101
|
"""Tracks work sessions across shell commands.
|
|
@@ -209,29 +209,21 @@ add-zsh-hook precmd _mem_precmd
|
|
|
209
209
|
"""
|
|
210
210
|
|
|
211
211
|
|
|
212
|
-
@cli.command()
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
def sync(keep_commands: int, keep_sessions: int) -> None:
|
|
216
|
-
"""Extract patterns and rotate old data."""
|
|
217
|
-
from rich.progress import Progress
|
|
218
|
-
|
|
219
|
-
from mem.patterns import sync_all_patterns
|
|
220
|
-
from mem import storage
|
|
221
|
-
|
|
222
|
-
# Pattern extraction
|
|
223
|
-
with Progress(console=console) as progress:
|
|
224
|
-
task = progress.add_task("Extracting patterns...", total=None)
|
|
225
|
-
new, updated = sync_all_patterns()
|
|
226
|
-
progress.update(task, completed=100, total=100)
|
|
212
|
+
@cli.command(name="_sync", hidden=True)
|
|
213
|
+
def sync_cmd() -> None:
|
|
214
|
+
"""Internal: background pattern extraction and data rotation.
|
|
227
215
|
|
|
228
|
-
|
|
216
|
+
Triggered automatically every 20 captured commands. Runs silently —
|
|
217
|
+
no output, no errors. Never called by the user directly.
|
|
218
|
+
"""
|
|
219
|
+
try:
|
|
220
|
+
from mem.patterns import sync_all_patterns
|
|
221
|
+
from mem import storage
|
|
229
222
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
console.print(f" Sessions older than {keep_sessions}d: {sess_removed} removed")
|
|
223
|
+
sync_all_patterns(silent=True)
|
|
224
|
+
storage.rotate()
|
|
225
|
+
except Exception:
|
|
226
|
+
pass # Background task — never surface errors
|
|
235
227
|
|
|
236
228
|
|
|
237
229
|
@cli.command()
|
|
@@ -118,24 +118,56 @@ async def _generalize_commands(tool: str, unique_commands: list[str]) -> dict[st
|
|
|
118
118
|
|
|
119
119
|
|
|
120
120
|
async def extract_patterns_for_tool(
|
|
121
|
-
tool: str,
|
|
121
|
+
tool: str,
|
|
122
|
+
commands: list[str],
|
|
123
|
+
already_processed: set[str] | None = None,
|
|
122
124
|
) -> PatternExtractionResult:
|
|
123
125
|
"""Extract abstract patterns from a list of concrete commands.
|
|
124
126
|
|
|
125
127
|
Strategy:
|
|
126
128
|
1. Deduplicate commands and count frequencies (code)
|
|
127
129
|
2. Generalize each unique command via LLM (if available)
|
|
130
|
+
— skips commands in already_processed (cache hit)
|
|
128
131
|
3. Aggregate frequencies by generalized pattern (code)
|
|
129
132
|
|
|
130
133
|
Falls back to simple frequency grouping if SDK is unavailable.
|
|
131
134
|
"""
|
|
135
|
+
if already_processed is None:
|
|
136
|
+
already_processed = set()
|
|
137
|
+
|
|
132
138
|
# Step 1: Count raw frequencies (code — fast and exact)
|
|
133
139
|
raw_freq = Counter(commands)
|
|
134
140
|
unique_cmds = list(raw_freq.keys())
|
|
135
141
|
|
|
136
142
|
if _apple_fm_available():
|
|
137
|
-
# Step 2: Generalize unique commands (LLM
|
|
138
|
-
|
|
143
|
+
# Step 2: Generalize only NEW unique commands (LLM)
|
|
144
|
+
new_cmds = [c for c in unique_cmds if c not in already_processed]
|
|
145
|
+
|
|
146
|
+
# Load existing patterns to reuse cached generalizations
|
|
147
|
+
existing_pf = storage.read_patterns(tool)
|
|
148
|
+
cached_map: dict[str, str] = {}
|
|
149
|
+
if existing_pf:
|
|
150
|
+
# Rebuild command->pattern map from existing data
|
|
151
|
+
for p in existing_pf.patterns:
|
|
152
|
+
cached_map[p.example] = p.pattern
|
|
153
|
+
|
|
154
|
+
if new_cmds:
|
|
155
|
+
new_map = await _generalize_commands(tool, new_cmds)
|
|
156
|
+
else:
|
|
157
|
+
new_map = {}
|
|
158
|
+
|
|
159
|
+
# Merge: cached + new
|
|
160
|
+
cmd_to_pattern: dict[str, str] = {}
|
|
161
|
+
for cmd in unique_cmds:
|
|
162
|
+
if cmd in new_map:
|
|
163
|
+
cmd_to_pattern[cmd] = new_map[cmd]
|
|
164
|
+
elif cmd in cached_map:
|
|
165
|
+
cmd_to_pattern[cmd] = cached_map[cmd]
|
|
166
|
+
else:
|
|
167
|
+
# Command was processed before but not in cache
|
|
168
|
+
# (e.g., its example was a different command for same pattern)
|
|
169
|
+
# Fall back to raw command
|
|
170
|
+
cmd_to_pattern[cmd] = cmd
|
|
139
171
|
|
|
140
172
|
# Step 3: Aggregate by pattern (code — exact counting)
|
|
141
173
|
pattern_freq: Counter[str] = Counter()
|
|
@@ -197,34 +229,59 @@ def run_pattern_extraction(tool: str) -> None:
|
|
|
197
229
|
|
|
198
230
|
Reads all commands starting with the tool name from storage,
|
|
199
231
|
runs extraction, and writes the result to patterns/<tool>.json.
|
|
232
|
+
|
|
233
|
+
Uses caching: only sends new (unprocessed) commands to the LLM.
|
|
234
|
+
Already-processed commands are tracked in the PatternFile.
|
|
200
235
|
"""
|
|
201
236
|
import asyncio
|
|
202
237
|
|
|
203
|
-
|
|
238
|
+
all_commands = [
|
|
204
239
|
cmd.command
|
|
205
240
|
for cmd in storage.read_all_commands()
|
|
206
241
|
if cmd.command.split()[0] == tool
|
|
207
242
|
]
|
|
208
243
|
|
|
209
|
-
if len(
|
|
244
|
+
if len(all_commands) < 5:
|
|
210
245
|
return # Not enough data for meaningful patterns
|
|
211
246
|
|
|
212
|
-
|
|
247
|
+
# Load existing pattern file for cache
|
|
248
|
+
existing = storage.read_patterns(tool)
|
|
249
|
+
already_processed: set[str] = set()
|
|
250
|
+
if existing and existing.processed_commands:
|
|
251
|
+
already_processed = set(existing.processed_commands)
|
|
252
|
+
|
|
253
|
+
# Only send new commands to the LLM
|
|
254
|
+
new_commands = [c for c in all_commands if c not in already_processed]
|
|
255
|
+
|
|
256
|
+
if not new_commands and existing:
|
|
257
|
+
return # Nothing new to process
|
|
258
|
+
|
|
259
|
+
# Extract patterns from ALL commands (new + old) but only generalize new ones
|
|
260
|
+
result = asyncio.run(
|
|
261
|
+
extract_patterns_for_tool(tool, all_commands, already_processed)
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
# Track all unique commands as processed
|
|
265
|
+
all_unique = list(set(all_commands))
|
|
213
266
|
|
|
214
267
|
pf = PatternFile(
|
|
215
268
|
tool=tool,
|
|
216
269
|
patterns=result.patterns,
|
|
217
270
|
last_updated=int(time.time()),
|
|
271
|
+
processed_commands=all_unique,
|
|
218
272
|
)
|
|
219
273
|
storage.write_patterns(pf)
|
|
220
274
|
|
|
221
275
|
|
|
222
|
-
def sync_all_patterns() -> tuple[int, int]:
|
|
276
|
+
def sync_all_patterns(silent: bool = False) -> tuple[int, int]:
|
|
223
277
|
"""Extract patterns for ALL tools with sufficient command history.
|
|
224
278
|
|
|
225
279
|
Detects unique tools (first token of each command), runs extraction
|
|
226
280
|
for each tool with >5 commands. Skips tools with insufficient data.
|
|
227
281
|
|
|
282
|
+
Args:
|
|
283
|
+
silent: If True, suppress all output (for background auto-sync).
|
|
284
|
+
|
|
228
285
|
Returns (new_patterns, updated_patterns) counts.
|
|
229
286
|
"""
|
|
230
287
|
# Collect all commands grouped by tool (first token)
|
|
@@ -249,7 +306,7 @@ def sync_all_patterns() -> tuple[int, int]:
|
|
|
249
306
|
else:
|
|
250
307
|
updated_count += 1
|
|
251
308
|
|
|
252
|
-
if not _apple_fm_available():
|
|
309
|
+
if not silent and not _apple_fm_available():
|
|
253
310
|
print(
|
|
254
311
|
"Tip: install AI support for smarter pattern extraction: "
|
|
255
312
|
"pip install cli-mem[ai]",
|
|
@@ -312,6 +312,36 @@ def forget_commands(query: str) -> int:
|
|
|
312
312
|
return removed
|
|
313
313
|
|
|
314
314
|
|
|
315
|
+
# --- Sync counter ---
|
|
316
|
+
|
|
317
|
+
SYNC_COUNTER_FILE = MEM_DIR / ".sync_counter"
|
|
318
|
+
SYNC_THRESHOLD = 20
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def read_sync_counter() -> int:
|
|
322
|
+
"""Read the number of captures since last sync."""
|
|
323
|
+
if not SYNC_COUNTER_FILE.exists():
|
|
324
|
+
return 0
|
|
325
|
+
try:
|
|
326
|
+
return int(SYNC_COUNTER_FILE.read_text(encoding="utf-8").strip())
|
|
327
|
+
except (ValueError, OSError):
|
|
328
|
+
return 0
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def increment_sync_counter() -> int:
|
|
332
|
+
"""Increment capture counter and return new value."""
|
|
333
|
+
count = read_sync_counter() + 1
|
|
334
|
+
ensure_dirs()
|
|
335
|
+
SYNC_COUNTER_FILE.write_text(str(count), encoding="utf-8")
|
|
336
|
+
return count
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def reset_sync_counter() -> None:
|
|
340
|
+
"""Reset capture counter after a sync."""
|
|
341
|
+
ensure_dirs()
|
|
342
|
+
SYNC_COUNTER_FILE.write_text("0", encoding="utf-8")
|
|
343
|
+
|
|
344
|
+
|
|
315
345
|
# --- Named Groups storage ---
|
|
316
346
|
|
|
317
347
|
|
|
@@ -313,6 +313,41 @@ def _normalize_var_name(name: str) -> str:
|
|
|
313
313
|
return result.strip("_")
|
|
314
314
|
|
|
315
315
|
|
|
316
|
+
def _extract_value_from_syntax(original_value: str, cmd: str) -> str:
|
|
317
|
+
"""Extract the actual secret value from flag=value or KEY=value syntax.
|
|
318
|
+
|
|
319
|
+
The AI sometimes returns 'GITHUB_TOKEN=ghp_abc...' or '--password=secret'
|
|
320
|
+
as the original_value. We need just the secret part so that .replace()
|
|
321
|
+
doesn't destroy the command structure.
|
|
322
|
+
"""
|
|
323
|
+
# Pattern: --flag=value or -f=value (CLI flag syntax)
|
|
324
|
+
flag_match = re.match(r"^--?[a-zA-Z][\w-]*=(.+)$", original_value)
|
|
325
|
+
if flag_match:
|
|
326
|
+
value = flag_match.group(1)
|
|
327
|
+
if value in cmd:
|
|
328
|
+
return value
|
|
329
|
+
|
|
330
|
+
# Pattern: ENV_VAR=value (environment variable assignment)
|
|
331
|
+
env_match = re.match(r"^[A-Z][A-Z0-9_]+=(.+)$", original_value)
|
|
332
|
+
if env_match:
|
|
333
|
+
value = env_match.group(1)
|
|
334
|
+
if value in cmd:
|
|
335
|
+
return value
|
|
336
|
+
|
|
337
|
+
return original_value
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _looks_like_hostname(value: str) -> bool:
|
|
341
|
+
"""Check if a value looks like a hostname or domain, not a credential."""
|
|
342
|
+
# Hostnames: words joined by dots, no special chars typical of secrets
|
|
343
|
+
if re.match(r"^[a-zA-Z0-9]([a-zA-Z0-9-]*\.)+[a-zA-Z]{2,}$", value):
|
|
344
|
+
return True
|
|
345
|
+
# IP addresses
|
|
346
|
+
if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", value):
|
|
347
|
+
return True
|
|
348
|
+
return False
|
|
349
|
+
|
|
350
|
+
|
|
316
351
|
def _deduplicate_detections(
|
|
317
352
|
detections: list[tuple[str, str, str]],
|
|
318
353
|
cmd: str,
|
|
@@ -325,30 +360,39 @@ def _deduplicate_detections(
|
|
|
325
360
|
- Very short values (< 8 chars, not plausible secrets)
|
|
326
361
|
- Duplicate original_value entries
|
|
327
362
|
- Values that are substrings of other detected values
|
|
363
|
+
|
|
364
|
+
Also extracts the actual secret from flag=value syntax when the AI
|
|
365
|
+
returns the whole expression as original_value.
|
|
328
366
|
"""
|
|
329
367
|
result: list[tuple[str, str, str]] = []
|
|
330
368
|
seen_values: set[str] = set()
|
|
331
369
|
|
|
332
370
|
for original_value, suggested_name, reason in detections:
|
|
371
|
+
# Extract actual value from flag=value or ENV=value syntax
|
|
372
|
+
value = _extract_value_from_syntax(original_value, cmd)
|
|
373
|
+
|
|
333
374
|
# Skip hallucinated values not in the command
|
|
334
|
-
if
|
|
375
|
+
if value not in cmd:
|
|
335
376
|
continue
|
|
336
377
|
# Skip URLs
|
|
337
|
-
if
|
|
378
|
+
if value.startswith(("http://", "https://")):
|
|
379
|
+
continue
|
|
380
|
+
# Skip hostnames and IP addresses
|
|
381
|
+
if _looks_like_hostname(value):
|
|
338
382
|
continue
|
|
339
383
|
# Skip very short values
|
|
340
|
-
if len(
|
|
384
|
+
if len(value) < 8:
|
|
341
385
|
continue
|
|
342
386
|
# Skip duplicates
|
|
343
|
-
if
|
|
387
|
+
if value in seen_values:
|
|
344
388
|
continue
|
|
345
389
|
# Normalize suggested name to UPPER_SNAKE_CASE
|
|
346
390
|
normalized = _normalize_var_name(suggested_name)
|
|
347
391
|
if not re.match(r"^[A-Z][A-Z0-9_]+$", normalized):
|
|
348
392
|
continue
|
|
349
393
|
|
|
350
|
-
seen_values.add(
|
|
351
|
-
result.append((
|
|
394
|
+
seen_values.add(value)
|
|
395
|
+
result.append((value, normalized, reason))
|
|
352
396
|
|
|
353
397
|
# Remove values that are substrings of other detected values
|
|
354
398
|
filtered: list[tuple[str, str, str]] = []
|
|
@@ -387,17 +431,26 @@ async def _detect_credentials_async(cmd: str) -> list[tuple[str, str, str]]:
|
|
|
387
431
|
prompt = (
|
|
388
432
|
"Analyze this shell command and find ONLY hardcoded sensitive values "
|
|
389
433
|
"that should be replaced with environment variables.\n\n"
|
|
434
|
+
"RULES FOR original_value:\n"
|
|
435
|
+
"- Return ONLY the literal secret value, not the flag or key name.\n"
|
|
436
|
+
"- For --password=mysecret, return 'mysecret' not '--password=mysecret'.\n"
|
|
437
|
+
"- For GITHUB_TOKEN=ghp_abc, return 'ghp_abc' not 'GITHUB_TOKEN=ghp_abc'.\n"
|
|
438
|
+
"- For postgres://user:pass@host, return 'pass' not the full URL.\n\n"
|
|
439
|
+
"RULES FOR suggested_name:\n"
|
|
440
|
+
"- Derive the name from the service or tool in the command.\n"
|
|
441
|
+
"- Examples: STRIPE_API_KEY, GITHUB_TOKEN, REDIS_PASSWORD, DB_PASSWORD.\n"
|
|
442
|
+
"- NEVER use generic prefixes like ACME_.\n\n"
|
|
390
443
|
"ONLY flag these types of values:\n"
|
|
391
|
-
"- API tokens or keys (long alphanumeric/base64 strings
|
|
392
|
-
"- Passwords passed inline (after --password, -p, etc.)\n"
|
|
444
|
+
"- API tokens or keys (long alphanumeric/base64 strings)\n"
|
|
445
|
+
"- Passwords passed inline (after --password, -p, -a, etc.)\n"
|
|
393
446
|
"- Bearer tokens in Authorization headers\n"
|
|
394
|
-
"-
|
|
447
|
+
"- Embedded passwords in connection strings\n\n"
|
|
395
448
|
"DO NOT flag:\n"
|
|
396
|
-
"- URLs, hostnames, or
|
|
397
|
-
"- The command name or
|
|
449
|
+
"- URLs, hostnames, IP addresses, or domain names\n"
|
|
450
|
+
"- The command name, subcommands, or usernames\n"
|
|
398
451
|
"- Short strings, regular arguments, or file paths\n"
|
|
399
|
-
"- Port numbers or
|
|
400
|
-
"If
|
|
452
|
+
"- Port numbers, known constants, or registry addresses\n\n"
|
|
453
|
+
"If NO credentials found, return an EMPTY list.\n\n"
|
|
401
454
|
f"Command: {cmd}"
|
|
402
455
|
)
|
|
403
456
|
|
|
@@ -21,6 +21,10 @@ def tmp_mem_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
|
|
21
21
|
monkeypatch.setattr(
|
|
22
22
|
storage, "GROUPS_GLOBAL_FILE", tmp_path / "groups" / "_global.json"
|
|
23
23
|
)
|
|
24
|
+
monkeypatch.setattr(
|
|
25
|
+
storage, "SYNC_COUNTER_FILE", tmp_path / ".sync_counter"
|
|
26
|
+
)
|
|
27
|
+
monkeypatch.setattr(storage, "VARS_FILE", tmp_path / "vars.json")
|
|
24
28
|
return tmp_path
|
|
25
29
|
|
|
26
30
|
|
|
@@ -1858,3 +1858,110 @@ class TestDeduplicateDetections:
|
|
|
1858
1858
|
detections = [(token, "AcmeApiToken", "JWT")]
|
|
1859
1859
|
result = _deduplicate_detections(detections, cmd)
|
|
1860
1860
|
assert result[0][1] == "ACME_API_TOKEN"
|
|
1861
|
+
|
|
1862
|
+
def test_extracts_value_from_flag_syntax(self):
|
|
1863
|
+
"""Bug A: AI returns '--password=secret' instead of just 'secret'."""
|
|
1864
|
+
from mem.variables import _deduplicate_detections
|
|
1865
|
+
|
|
1866
|
+
cmd = "mysql -u admin --password=S3cretP@ssw0rd123 mydb"
|
|
1867
|
+
detections = [("--password=S3cretP@ssw0rd123", "DB_PASSWORD", "password")]
|
|
1868
|
+
result = _deduplicate_detections(detections, cmd)
|
|
1869
|
+
assert len(result) == 1
|
|
1870
|
+
assert result[0][0] == "S3cretP@ssw0rd123"
|
|
1871
|
+
|
|
1872
|
+
def test_extracts_value_from_env_var_syntax(self):
|
|
1873
|
+
"""Bug A: AI returns 'GITHUB_TOKEN=ghp_abc...' instead of just the token."""
|
|
1874
|
+
from mem.variables import _deduplicate_detections
|
|
1875
|
+
|
|
1876
|
+
token = "ghp_abcdefghijklmnopqrstuvwxyz123456"
|
|
1877
|
+
cmd = f"GITHUB_TOKEN={token} gh pr create"
|
|
1878
|
+
detections = [(f"GITHUB_TOKEN={token}", "GITHUB_TOKEN", "API token")]
|
|
1879
|
+
result = _deduplicate_detections(detections, cmd)
|
|
1880
|
+
assert len(result) == 1
|
|
1881
|
+
assert result[0][0] == token
|
|
1882
|
+
|
|
1883
|
+
def test_filters_hostnames(self):
|
|
1884
|
+
"""Bug C: hostnames like registry.ghcr.io are not credentials."""
|
|
1885
|
+
from mem.variables import _deduplicate_detections
|
|
1886
|
+
|
|
1887
|
+
cmd = "docker login -u deploy -p secret123456789 registry.ghcr.io"
|
|
1888
|
+
detections = [
|
|
1889
|
+
("secret123456789", "REGISTRY_TOKEN", "token"),
|
|
1890
|
+
("registry.ghcr.io", "REGISTRY_URL", "hostname"),
|
|
1891
|
+
]
|
|
1892
|
+
result = _deduplicate_detections(detections, cmd)
|
|
1893
|
+
assert len(result) == 1
|
|
1894
|
+
assert result[0][0] == "secret123456789"
|
|
1895
|
+
|
|
1896
|
+
def test_filters_ip_addresses(self):
|
|
1897
|
+
from mem.variables import _deduplicate_detections
|
|
1898
|
+
|
|
1899
|
+
cmd = "redis-cli -h 10.0.1.50 -a MyR3d1sPassword123 PING"
|
|
1900
|
+
detections = [
|
|
1901
|
+
("MyR3d1sPassword123", "REDIS_PASSWORD", "password"),
|
|
1902
|
+
("10.0.1.50", "REDIS_HOST", "IP address"),
|
|
1903
|
+
]
|
|
1904
|
+
result = _deduplicate_detections(detections, cmd)
|
|
1905
|
+
assert len(result) == 1
|
|
1906
|
+
assert result[0][0] == "MyR3d1sPassword123"
|
|
1907
|
+
|
|
1908
|
+
|
|
1909
|
+
class TestExtractValueFromSyntax:
|
|
1910
|
+
"""Unit tests for _extract_value_from_syntax."""
|
|
1911
|
+
|
|
1912
|
+
def test_cli_flag_long(self):
|
|
1913
|
+
from mem.variables import _extract_value_from_syntax
|
|
1914
|
+
|
|
1915
|
+
cmd = "mysql --password=secret123 mydb"
|
|
1916
|
+
assert _extract_value_from_syntax("--password=secret123", cmd) == "secret123"
|
|
1917
|
+
|
|
1918
|
+
def test_cli_flag_short(self):
|
|
1919
|
+
from mem.variables import _extract_value_from_syntax
|
|
1920
|
+
|
|
1921
|
+
cmd = "tool -p=myvalue123 arg"
|
|
1922
|
+
assert _extract_value_from_syntax("-p=myvalue123", cmd) == "myvalue123"
|
|
1923
|
+
|
|
1924
|
+
def test_env_var_assignment(self):
|
|
1925
|
+
from mem.variables import _extract_value_from_syntax
|
|
1926
|
+
|
|
1927
|
+
cmd = "GITHUB_TOKEN=ghp_abc123 gh pr create"
|
|
1928
|
+
assert _extract_value_from_syntax("GITHUB_TOKEN=ghp_abc123", cmd) == "ghp_abc123"
|
|
1929
|
+
|
|
1930
|
+
def test_plain_value_unchanged(self):
|
|
1931
|
+
from mem.variables import _extract_value_from_syntax
|
|
1932
|
+
|
|
1933
|
+
cmd = "curl -H 'Bearer eyJtoken'"
|
|
1934
|
+
assert _extract_value_from_syntax("eyJtoken", cmd) == "eyJtoken"
|
|
1935
|
+
|
|
1936
|
+
def test_value_not_in_cmd_returns_original(self):
|
|
1937
|
+
from mem.variables import _extract_value_from_syntax
|
|
1938
|
+
|
|
1939
|
+
cmd = "echo hello"
|
|
1940
|
+
assert _extract_value_from_syntax("--flag=nothere", cmd) == "--flag=nothere"
|
|
1941
|
+
|
|
1942
|
+
|
|
1943
|
+
class TestLooksLikeHostname:
|
|
1944
|
+
def test_domain(self):
|
|
1945
|
+
from mem.variables import _looks_like_hostname
|
|
1946
|
+
|
|
1947
|
+
assert _looks_like_hostname("registry.ghcr.io") is True
|
|
1948
|
+
|
|
1949
|
+
def test_subdomain(self):
|
|
1950
|
+
from mem.variables import _looks_like_hostname
|
|
1951
|
+
|
|
1952
|
+
assert _looks_like_hostname("api.prod.internal.company.com") is True
|
|
1953
|
+
|
|
1954
|
+
def test_ip_address(self):
|
|
1955
|
+
from mem.variables import _looks_like_hostname
|
|
1956
|
+
|
|
1957
|
+
assert _looks_like_hostname("10.0.1.50") is True
|
|
1958
|
+
|
|
1959
|
+
def test_token_not_hostname(self):
|
|
1960
|
+
from mem.variables import _looks_like_hostname
|
|
1961
|
+
|
|
1962
|
+
assert _looks_like_hostname("eyJhbGciOiJIUzI1NiJ9") is False
|
|
1963
|
+
|
|
1964
|
+
def test_password_not_hostname(self):
|
|
1965
|
+
from mem.variables import _looks_like_hostname
|
|
1966
|
+
|
|
1967
|
+
assert _looks_like_hostname("S3cretP@ssw0rd") is False
|
|
@@ -463,7 +463,7 @@ class TestDeduplication:
|
|
|
463
463
|
|
|
464
464
|
class TestSyncAllPatterns:
|
|
465
465
|
def test_sync_warns_without_sdk(self, tmp_mem_dir, capsys):
|
|
466
|
-
"""sync_all_patterns prints warning when SDK is unavailable."""
|
|
466
|
+
"""sync_all_patterns prints warning when SDK is unavailable (non-silent)."""
|
|
467
467
|
now = int(time.time())
|
|
468
468
|
for i in range(6):
|
|
469
469
|
storage.append_command(
|
|
@@ -481,6 +481,25 @@ class TestSyncAllPatterns:
|
|
|
481
481
|
captured = capsys.readouterr()
|
|
482
482
|
assert "pip install cli-mem[ai]" in captured.err
|
|
483
483
|
|
|
484
|
+
def test_sync_silent_no_output(self, tmp_mem_dir, capsys):
|
|
485
|
+
"""sync_all_patterns(silent=True) produces no output."""
|
|
486
|
+
now = int(time.time())
|
|
487
|
+
for i in range(6):
|
|
488
|
+
storage.append_command(
|
|
489
|
+
make_command(
|
|
490
|
+
command=f"make target-{i}",
|
|
491
|
+
ts=now,
|
|
492
|
+
repo="/Users/test/projects/myapp",
|
|
493
|
+
)
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
with patch.object(patterns, "_apple_fm_available", return_value=False):
|
|
497
|
+
new, updated = patterns.sync_all_patterns(silent=True)
|
|
498
|
+
|
|
499
|
+
assert new == 1
|
|
500
|
+
captured = capsys.readouterr()
|
|
501
|
+
assert captured.err == ""
|
|
502
|
+
|
|
484
503
|
def test_sync_skips_tools_below_threshold(self, tmp_mem_dir):
|
|
485
504
|
"""Tools with <5 commands are skipped entirely."""
|
|
486
505
|
now = int(time.time())
|
|
@@ -524,6 +543,93 @@ class TestSyncAllPatterns:
|
|
|
524
543
|
assert storage.read_patterns("tool-c") is None
|
|
525
544
|
|
|
526
545
|
|
|
546
|
+
class TestPatternCaching:
|
|
547
|
+
"""Verify that already-processed commands skip the LLM."""
|
|
548
|
+
|
|
549
|
+
@pytest.mark.asyncio
|
|
550
|
+
async def test_cached_commands_skip_llm(self):
|
|
551
|
+
"""Commands in already_processed set should not trigger LLM calls."""
|
|
552
|
+
call_count = 0
|
|
553
|
+
|
|
554
|
+
async def _counting_respond(prompt: str, generating=None):
|
|
555
|
+
nonlocal call_count
|
|
556
|
+
call_count += 1
|
|
557
|
+
for line in prompt.splitlines():
|
|
558
|
+
if line.startswith("Command:"):
|
|
559
|
+
cmd = line.split("Command:", 1)[1].strip()
|
|
560
|
+
return _make_mock_generalized(f"{cmd} <generalized>")
|
|
561
|
+
return _make_mock_generalized("unknown")
|
|
562
|
+
|
|
563
|
+
mock_session = MockSession(respond_fn=_counting_respond)
|
|
564
|
+
|
|
565
|
+
with (
|
|
566
|
+
patch.object(patterns, "_apple_fm_available", return_value=True),
|
|
567
|
+
patch("mem.patterns._get_generable_types", return_value=MagicMock()),
|
|
568
|
+
patch("apple_fm_sdk.LanguageModelSession", return_value=mock_session),
|
|
569
|
+
):
|
|
570
|
+
# First call: 3 unique commands, all new
|
|
571
|
+
commands = ["git status", "git log", "git diff", "git status", "git log"]
|
|
572
|
+
await patterns.extract_patterns_for_tool("git", commands)
|
|
573
|
+
|
|
574
|
+
assert call_count == 3 # 3 unique commands
|
|
575
|
+
|
|
576
|
+
# Second call with cache: only 1 new command
|
|
577
|
+
call_count = 0
|
|
578
|
+
already_done = {"git status", "git log", "git diff"}
|
|
579
|
+
|
|
580
|
+
with (
|
|
581
|
+
patch.object(patterns, "_apple_fm_available", return_value=True),
|
|
582
|
+
patch("mem.patterns._get_generable_types", return_value=MagicMock()),
|
|
583
|
+
patch("apple_fm_sdk.LanguageModelSession", return_value=mock_session),
|
|
584
|
+
):
|
|
585
|
+
commands2 = commands + ["git push"]
|
|
586
|
+
await patterns.extract_patterns_for_tool(
|
|
587
|
+
"git", commands2, already_done
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
assert call_count == 1 # Only "git push" is new
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
class TestAutoSync:
|
|
594
|
+
"""Verify the sync counter and auto-trigger logic."""
|
|
595
|
+
|
|
596
|
+
def test_counter_increment(self, tmp_mem_dir):
|
|
597
|
+
assert storage.read_sync_counter() == 0
|
|
598
|
+
assert storage.increment_sync_counter() == 1
|
|
599
|
+
assert storage.increment_sync_counter() == 2
|
|
600
|
+
assert storage.read_sync_counter() == 2
|
|
601
|
+
|
|
602
|
+
def test_counter_reset(self, tmp_mem_dir):
|
|
603
|
+
storage.increment_sync_counter()
|
|
604
|
+
storage.increment_sync_counter()
|
|
605
|
+
storage.reset_sync_counter()
|
|
606
|
+
assert storage.read_sync_counter() == 0
|
|
607
|
+
|
|
608
|
+
def test_capture_triggers_sync_at_threshold(self, tmp_mem_dir):
|
|
609
|
+
"""After SYNC_THRESHOLD captures, _spawn_background_sync is called."""
|
|
610
|
+
from mem import capture
|
|
611
|
+
|
|
612
|
+
with (
|
|
613
|
+
patch.object(storage, "SYNC_THRESHOLD", 3),
|
|
614
|
+
patch.object(capture, "_spawn_background_sync") as mock_spawn,
|
|
615
|
+
patch.object(capture, "get_git_repo", return_value=None),
|
|
616
|
+
):
|
|
617
|
+
capture.capture_command("cmd1", "/tmp", 0, 100)
|
|
618
|
+
capture.capture_command("cmd2", "/tmp", 0, 100)
|
|
619
|
+
assert mock_spawn.call_count == 0
|
|
620
|
+
|
|
621
|
+
capture.capture_command("cmd3", "/tmp", 0, 100)
|
|
622
|
+
assert mock_spawn.call_count == 1
|
|
623
|
+
|
|
624
|
+
# Counter reset, so next 3 should trigger again
|
|
625
|
+
capture.capture_command("cmd4", "/tmp", 0, 100)
|
|
626
|
+
capture.capture_command("cmd5", "/tmp", 0, 100)
|
|
627
|
+
assert mock_spawn.call_count == 1
|
|
628
|
+
|
|
629
|
+
capture.capture_command("cmd6", "/tmp", 0, 100)
|
|
630
|
+
assert mock_spawn.call_count == 2
|
|
631
|
+
|
|
632
|
+
|
|
527
633
|
# ---------------------------------------------------------------------------
|
|
528
634
|
# Test cases: Session summary generation
|
|
529
635
|
# ---------------------------------------------------------------------------
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|