synap-git 1.2.3__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {synap_git-1.2.3 → synap_git-2.0.0}/CHANGELOG.md +11 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/PKG-INFO +5 -5
- {synap_git-1.2.3 → synap_git-2.0.0}/README.md +4 -4
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/__init__.py +1 -1
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/engine.py +123 -22
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/scanner.py +36 -27
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/parser/registry.py +32 -22
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/gemini.py +53 -47
- synap_git-2.0.0/src/synap_git/provider/ollama.py +119 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/retrieval/engine.py +144 -44
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/storage/sqlite.py +114 -13
- synap_git-1.2.3/src/synap_git/provider/ollama.py +0 -140
- {synap_git-1.2.3 → synap_git-2.0.0}/.gitignore +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/.synap.example/README.md +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/LICENSE.md +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/pyproject.toml +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/api/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/api/app.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/api/static/index.html +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/cli/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/cli/__main__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/cli/main.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/config.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/diagnostics/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/diagnostics/logger.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/diagnostics/tracing.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/embeddings/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/git/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/git/state.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/daemon.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/wiki.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/mcp/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/mcp/server.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/parser/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/anthropic.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/base.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/factory.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/openai.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/openrouter.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/py.typed +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/retrieval/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/retrieval/memory.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/storage/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/utils/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/utils/serialization.py +0 -0
|
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [2.0.0] - 2026-05-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **.synapignore Support**: Implemented context-aware filtering to prevent traversing symlinks, binary files, and files listed in `.synapignore` or `.gitignore`.
|
|
12
|
+
- **FTS5 Integration**: Shifted semantic full-text search directly to SQLite FTS5 with bounding limits (`LIMIT 50`) for `O(1)` lexical retrieval scaling on 10,000+ file monorepos.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
- **LLM Hangs in Test Suite**: Isolated testing environment by strictly bypassing real LLM provider configs (`SYNAP_LLM_PROVIDER=""`) preventing test CI suite hangs.
|
|
16
|
+
- **Repository Path Scoping in Hybrid Engine**: Fixed a bug where `repo_path` initialization mistakenly truncated the path to `/tmp` in tests, failing contextual snippet aggregation.
|
|
17
|
+
- **Unbounded BM25 Expansion**: Bounded full-text search token extraction and BM25 results, preventing out-of-memory cascading in CTE traversal during monorepo indexing.
|
|
18
|
+
|
|
8
19
|
## [1.2.3] - 2026-05-28
|
|
9
20
|
|
|
10
21
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: synap-git
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Persistent structural context infrastructure for AI coding agents.
|
|
5
5
|
Project-URL: Homepage, https://github.com/saahilpal/synap-git
|
|
6
6
|
Project-URL: Repository, https://github.com/saahilpal/synap-git
|
|
@@ -52,10 +52,10 @@ Description-Content-Type: text/markdown
|
|
|
52
52
|
|
|
53
53
|
Local Git-aware structural context engine for AI coding agents.
|
|
54
54
|
|
|
55
|
-
[](https://
|
|
57
|
-
[](https://
|
|
58
|
-
[](https://github.com/saahilpal/synapse/actions)
|
|
56
|
+
[](https://pypi.org/project/synap-git/)
|
|
57
|
+
[](https://pypi.org/project/synap-git/)
|
|
58
|
+
[](LICENSE.md)
|
|
59
59
|
|
|
60
60
|
## The Problem
|
|
61
61
|
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
Local Git-aware structural context engine for AI coding agents.
|
|
4
4
|
|
|
5
|
-
[](https://
|
|
7
|
-
[](https://
|
|
8
|
-
[](https://github.com/saahilpal/synapse/actions)
|
|
6
|
+
[](https://pypi.org/project/synap-git/)
|
|
7
|
+
[](https://pypi.org/project/synap-git/)
|
|
8
|
+
[](LICENSE.md)
|
|
9
9
|
|
|
10
10
|
## The Problem
|
|
11
11
|
|
|
@@ -69,6 +69,7 @@ class SynapRuntime:
|
|
|
69
69
|
self.parser_registry = CodeParserRegistry()
|
|
70
70
|
self.llm_provider = get_llm_provider(settings)
|
|
71
71
|
self.retrieval_engine = HybridRetrievalEngine(
|
|
72
|
+
repo_path=self.settings.repository_path,
|
|
72
73
|
store=self.store,
|
|
73
74
|
llm_provider=self.llm_provider,
|
|
74
75
|
trace_store=self.trace_store,
|
|
@@ -244,6 +245,44 @@ class SynapRuntime:
|
|
|
244
245
|
|
|
245
246
|
return commit
|
|
246
247
|
|
|
248
|
+
def _schedule_embedding(
|
|
249
|
+
self, file_id: str, symbols: list[dict[str, Any]], content_hash: str
|
|
250
|
+
) -> None:
|
|
251
|
+
provider = self.llm_provider
|
|
252
|
+
if not provider:
|
|
253
|
+
return
|
|
254
|
+
|
|
255
|
+
def _worker() -> None:
|
|
256
|
+
import structlog
|
|
257
|
+
|
|
258
|
+
logger = structlog.get_logger()
|
|
259
|
+
try:
|
|
260
|
+
batches = [symbols[i : i + 50] for i in range(0, len(symbols), 50)]
|
|
261
|
+
model_name = getattr(provider, "default_model", "unknown")
|
|
262
|
+
for batch in batches:
|
|
263
|
+
for sym in batch:
|
|
264
|
+
try:
|
|
265
|
+
# Avoid re-embedding if unchanged?
|
|
266
|
+
text = sym["name"] + " " + sym.get("metadata", {}).get("docstring", "")
|
|
267
|
+
vector = provider.embed(text)
|
|
268
|
+
self.store.put_embedding(
|
|
269
|
+
sym["symbol_id"], model_name, "1.0", "1.0", vector, content_hash
|
|
270
|
+
)
|
|
271
|
+
except Exception as e:
|
|
272
|
+
logger.warning("background_embed_failed_for_symbol", error=str(e))
|
|
273
|
+
except Exception as e:
|
|
274
|
+
logger.warning("background_embed_failed", error=str(e))
|
|
275
|
+
|
|
276
|
+
try:
|
|
277
|
+
import asyncio
|
|
278
|
+
|
|
279
|
+
loop = asyncio.get_running_loop()
|
|
280
|
+
loop.create_task(asyncio.to_thread(_worker))
|
|
281
|
+
except RuntimeError:
|
|
282
|
+
import threading
|
|
283
|
+
|
|
284
|
+
threading.Thread(target=_worker, daemon=True).start()
|
|
285
|
+
|
|
247
286
|
def _first_run_index(self, git_state: GitState) -> str | None:
|
|
248
287
|
self.logger.info("first_run_indexing_started", branch=git_state.effective_branch)
|
|
249
288
|
import time
|
|
@@ -254,16 +293,9 @@ class SynapRuntime:
|
|
|
254
293
|
repository_path=self.settings.repository_path,
|
|
255
294
|
max_file_bytes=self.settings.max_file_bytes,
|
|
256
295
|
)
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
files_to_parse = []
|
|
260
|
-
for file_info in scan.files:
|
|
261
|
-
files_to_parse.append((file_info.path, file_info.relative_path, file_info.content))
|
|
262
|
-
|
|
263
|
-
num_files = len(files_to_parse)
|
|
296
|
+
num_files = scanner.count_files()
|
|
264
297
|
chunk_size = 500
|
|
265
298
|
parsed_results = []
|
|
266
|
-
|
|
267
299
|
# Parallel parsing using ProcessPoolExecutor
|
|
268
300
|
import multiprocessing
|
|
269
301
|
from concurrent.futures import ProcessPoolExecutor
|
|
@@ -301,16 +333,22 @@ class SynapRuntime:
|
|
|
301
333
|
task = progress.add_task("[yellow]Parsing codebase...[/yellow]", total=num_files)
|
|
302
334
|
|
|
303
335
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
304
|
-
|
|
305
|
-
|
|
336
|
+
from itertools import islice
|
|
337
|
+
|
|
338
|
+
file_iterator = scanner.scan()
|
|
339
|
+
while True:
|
|
340
|
+
chunk_files = list(islice(file_iterator, chunk_size))
|
|
341
|
+
if not chunk_files:
|
|
342
|
+
break
|
|
343
|
+
|
|
344
|
+
file_info_map = {f.relative_path: f for f in chunk_files}
|
|
345
|
+
chunk = [(f.path, f.relative_path, f.content) for f in chunk_files]
|
|
306
346
|
futures = [executor.submit(_parse_worker, item) for item in chunk]
|
|
307
347
|
chunk_results = [f.result() for f in futures]
|
|
308
348
|
|
|
309
349
|
for res in chunk_results:
|
|
310
350
|
rel_path = res["path"]
|
|
311
|
-
matching_info =
|
|
312
|
-
(f for f in scan.files if f.relative_path == rel_path), None
|
|
313
|
-
)
|
|
351
|
+
matching_info = file_info_map.get(rel_path)
|
|
314
352
|
git_oid = matching_info.git_oid if matching_info else ""
|
|
315
353
|
content_hash = matching_info.content_hash if matching_info else ""
|
|
316
354
|
|
|
@@ -328,6 +366,7 @@ class SynapRuntime:
|
|
|
328
366
|
language=res["language"],
|
|
329
367
|
symbols=res["symbols"],
|
|
330
368
|
)
|
|
369
|
+
self._schedule_embedding(file_id_hash, res["symbols"], content_hash)
|
|
331
370
|
self.store.enqueue_wiki(rel_path)
|
|
332
371
|
parsed_results.append((file_id_hash, rel_path, res["imports"]))
|
|
333
372
|
|
|
@@ -336,20 +375,26 @@ class SynapRuntime:
|
|
|
336
375
|
self._resolve_and_insert_edges(parsed_results)
|
|
337
376
|
parsed_results.clear()
|
|
338
377
|
|
|
339
|
-
progress.advance(task, len(
|
|
378
|
+
progress.advance(task, len(chunk_files))
|
|
340
379
|
del chunk_results
|
|
341
380
|
else:
|
|
342
381
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
343
|
-
|
|
344
|
-
|
|
382
|
+
from itertools import islice
|
|
383
|
+
|
|
384
|
+
file_iterator = scanner.scan()
|
|
385
|
+
while True:
|
|
386
|
+
chunk_files = list(islice(file_iterator, chunk_size))
|
|
387
|
+
if not chunk_files:
|
|
388
|
+
break
|
|
389
|
+
|
|
390
|
+
file_info_map = {f.relative_path: f for f in chunk_files}
|
|
391
|
+
chunk = [(f.path, f.relative_path, f.content) for f in chunk_files]
|
|
345
392
|
futures = [executor.submit(_parse_worker, item) for item in chunk]
|
|
346
393
|
chunk_results = [f.result() for f in futures]
|
|
347
394
|
|
|
348
395
|
for res in chunk_results:
|
|
349
396
|
rel_path = res["path"]
|
|
350
|
-
matching_info =
|
|
351
|
-
(f for f in scan.files if f.relative_path == rel_path), None
|
|
352
|
-
)
|
|
397
|
+
matching_info = file_info_map.get(rel_path)
|
|
353
398
|
git_oid = matching_info.git_oid if matching_info else ""
|
|
354
399
|
content_hash = matching_info.content_hash if matching_info else ""
|
|
355
400
|
|
|
@@ -367,6 +412,7 @@ class SynapRuntime:
|
|
|
367
412
|
language=res["language"],
|
|
368
413
|
symbols=res["symbols"],
|
|
369
414
|
)
|
|
415
|
+
self._schedule_embedding(file_id_hash, res["symbols"], content_hash)
|
|
370
416
|
self.store.enqueue_wiki(rel_path)
|
|
371
417
|
parsed_results.append((file_id_hash, rel_path, res["imports"]))
|
|
372
418
|
|
|
@@ -426,6 +472,7 @@ class SynapRuntime:
|
|
|
426
472
|
|
|
427
473
|
added_or_modified = []
|
|
428
474
|
deleted = []
|
|
475
|
+
renames = []
|
|
429
476
|
for line in res.stdout.splitlines():
|
|
430
477
|
if not line.strip():
|
|
431
478
|
continue
|
|
@@ -433,13 +480,41 @@ class SynapRuntime:
|
|
|
433
480
|
if len(parts) >= 2:
|
|
434
481
|
status = parts[0]
|
|
435
482
|
if status.startswith("R"):
|
|
436
|
-
|
|
483
|
+
renames.append((parts[1], parts[2]))
|
|
437
484
|
added_or_modified.append(parts[2])
|
|
438
485
|
elif status == "D":
|
|
439
486
|
deleted.append(parts[1])
|
|
440
487
|
else:
|
|
441
488
|
added_or_modified.append(parts[1])
|
|
442
489
|
|
|
490
|
+
import hashlib
|
|
491
|
+
|
|
492
|
+
# Apply renames to preserve identity
|
|
493
|
+
with self.store.connect() as conn:
|
|
494
|
+
for old_path, new_path in renames:
|
|
495
|
+
file_row = conn.execute(
|
|
496
|
+
"SELECT file_id, content_hash FROM files WHERE path = ?", (old_path,)
|
|
497
|
+
).fetchone()
|
|
498
|
+
if file_row:
|
|
499
|
+
old_file_id = file_row["file_id"]
|
|
500
|
+
content_hash = file_row["content_hash"]
|
|
501
|
+
new_file_id = hashlib.sha256(
|
|
502
|
+
(new_path + content_hash).encode("utf-8")
|
|
503
|
+
).hexdigest()
|
|
504
|
+
|
|
505
|
+
conn.execute("PRAGMA foreign_keys=OFF")
|
|
506
|
+
try:
|
|
507
|
+
conn.execute(
|
|
508
|
+
"UPDATE files SET file_id = ?, path = ? WHERE file_id = ?",
|
|
509
|
+
(new_file_id, new_path, old_file_id),
|
|
510
|
+
)
|
|
511
|
+
conn.execute(
|
|
512
|
+
"UPDATE symbols SET file_id = ? WHERE file_id = ?",
|
|
513
|
+
(new_file_id, old_file_id),
|
|
514
|
+
)
|
|
515
|
+
finally:
|
|
516
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
517
|
+
|
|
443
518
|
# Filter added/modified files
|
|
444
519
|
scanner = RepositoryScanner(
|
|
445
520
|
repository_path=self.settings.repository_path,
|
|
@@ -549,6 +624,7 @@ class SynapRuntime:
|
|
|
549
624
|
language=parse_result.language,
|
|
550
625
|
symbols=symbols_list,
|
|
551
626
|
)
|
|
627
|
+
self._schedule_embedding(file_id_hash, symbols_list, content_hash)
|
|
552
628
|
|
|
553
629
|
self.store.set_wiki_status(rel_path, None, "stale")
|
|
554
630
|
self.store.enqueue_wiki(rel_path)
|
|
@@ -701,11 +777,23 @@ class SynapRuntime:
|
|
|
701
777
|
for row in rows:
|
|
702
778
|
fts_symbols_by_name.setdefault(row["name"], []).append(dict(row))
|
|
703
779
|
|
|
780
|
+
target_symbols_dict: dict[str, dict[str, list[dict[str, Any]]]] = {}
|
|
781
|
+
for t_file_id, syms in target_symbols_by_file.items():
|
|
782
|
+
name_map: dict[str, Any] = {}
|
|
783
|
+
for sym in syms:
|
|
784
|
+
name_map.setdefault(sym["name"], []).append(sym)
|
|
785
|
+
target_symbols_dict[t_file_id] = name_map
|
|
786
|
+
|
|
704
787
|
for file_id, rel_path, imports in parsed_results:
|
|
705
788
|
file_symbols = file_symbols_map.get(file_id, [])
|
|
706
789
|
current_path = Path(rel_path)
|
|
790
|
+
edges_added_for_file = 0
|
|
791
|
+
max_edges_per_file = 1000
|
|
707
792
|
|
|
708
793
|
for imp in imports:
|
|
794
|
+
if edges_added_for_file >= max_edges_per_file:
|
|
795
|
+
break
|
|
796
|
+
|
|
709
797
|
target_file_id = None
|
|
710
798
|
if ":" in imp:
|
|
711
799
|
module_part, target_name = imp.split(":", 1)
|
|
@@ -728,15 +816,27 @@ class SynapRuntime:
|
|
|
728
816
|
else:
|
|
729
817
|
target_file_id = module_key_to_file_id.get(module_part)
|
|
730
818
|
|
|
819
|
+
# Fuzzy fallback
|
|
820
|
+
if not target_file_id:
|
|
821
|
+
for mk, fid in module_key_to_file_id.items():
|
|
822
|
+
if mk.endswith(module_part) or mk.endswith(f".{module_part}"):
|
|
823
|
+
target_file_id = fid
|
|
824
|
+
break
|
|
825
|
+
|
|
731
826
|
target_symbols = []
|
|
732
827
|
if target_file_id:
|
|
733
|
-
|
|
734
|
-
|
|
828
|
+
target_symbols = target_symbols_dict.get(target_file_id, {}).get(
|
|
829
|
+
target_name, []
|
|
830
|
+
)
|
|
735
831
|
else:
|
|
736
832
|
target_symbols = fts_symbols_by_name.get(target_name, [])
|
|
737
833
|
|
|
738
834
|
for ts in target_symbols:
|
|
835
|
+
if edges_added_for_file >= max_edges_per_file:
|
|
836
|
+
break
|
|
739
837
|
for fs in file_symbols:
|
|
838
|
+
if edges_added_for_file >= max_edges_per_file:
|
|
839
|
+
break
|
|
740
840
|
if fs["symbol_id"] == ts["symbol_id"]:
|
|
741
841
|
continue
|
|
742
842
|
edge_id = stable_hash(
|
|
@@ -747,6 +847,7 @@ class SynapRuntime:
|
|
|
747
847
|
}
|
|
748
848
|
)
|
|
749
849
|
edges_to_insert.append((edge_id, fs["symbol_id"], ts["symbol_id"]))
|
|
850
|
+
edges_added_for_file += 1
|
|
750
851
|
|
|
751
852
|
if edges_to_insert:
|
|
752
853
|
conn.executemany(
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import asyncio
|
|
4
3
|
import hashlib
|
|
5
4
|
import json
|
|
6
5
|
import re
|
|
7
6
|
import subprocess
|
|
8
7
|
import tomllib
|
|
8
|
+
from collections.abc import Iterator
|
|
9
9
|
from dataclasses import dataclass, field
|
|
10
10
|
from enum import StrEnum
|
|
11
11
|
from pathlib import Path
|
|
@@ -196,6 +196,7 @@ class RepositoryScanner:
|
|
|
196
196
|
self.excludes = excludes or DEFAULT_EXCLUDES
|
|
197
197
|
self.max_file_bytes = max_file_bytes
|
|
198
198
|
self.gitignore = None
|
|
199
|
+
self.synapignore = None
|
|
199
200
|
|
|
200
201
|
gitignore_path = self.repository_path / ".gitignore"
|
|
201
202
|
if gitignore_path.exists():
|
|
@@ -205,13 +206,15 @@ class RepositoryScanner:
|
|
|
205
206
|
except Exception as e:
|
|
206
207
|
logger.warning("failed_to_load_gitignore", error=str(e))
|
|
207
208
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
209
|
+
synapignore_path = self.repository_path / ".synapignore"
|
|
210
|
+
if synapignore_path.exists():
|
|
211
|
+
try:
|
|
212
|
+
lines = synapignore_path.read_text(encoding="utf-8").splitlines()
|
|
213
|
+
self.synapignore = GitIgnoreSpec(self.repository_path, lines)
|
|
214
|
+
except Exception as e:
|
|
215
|
+
logger.warning("failed_to_load_synapignore", error=str(e))
|
|
214
216
|
|
|
217
|
+
def _get_paths(self) -> list[Path]:
|
|
215
218
|
paths: list[Path] = []
|
|
216
219
|
try:
|
|
217
220
|
result = subprocess.run( # noqa: S603
|
|
@@ -227,10 +230,29 @@ class RepositoryScanner:
|
|
|
227
230
|
except UnicodeDecodeError:
|
|
228
231
|
pass
|
|
229
232
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
230
|
-
# Fallback to rglob if not a git repository
|
|
231
233
|
paths = list(self.repository_path.rglob("*"))
|
|
234
|
+
return paths
|
|
235
|
+
|
|
236
|
+
def count_files(self) -> int:
|
|
237
|
+
count = 0
|
|
238
|
+
for path in self._get_paths():
|
|
239
|
+
if not path.is_file():
|
|
240
|
+
continue
|
|
241
|
+
if self._excluded(path):
|
|
242
|
+
continue
|
|
243
|
+
try:
|
|
244
|
+
stat = path.stat()
|
|
245
|
+
if stat.st_size > self.max_file_bytes:
|
|
246
|
+
continue
|
|
247
|
+
except OSError:
|
|
248
|
+
continue
|
|
249
|
+
if _is_binary_file(path):
|
|
250
|
+
continue
|
|
251
|
+
count += 1
|
|
252
|
+
return count
|
|
232
253
|
|
|
233
|
-
|
|
254
|
+
def scan(self) -> Iterator[FileObservation]:
|
|
255
|
+
for path in self._get_paths():
|
|
234
256
|
if not path.is_file():
|
|
235
257
|
continue
|
|
236
258
|
if self._excluded(path):
|
|
@@ -250,11 +272,13 @@ class RepositoryScanner:
|
|
|
250
272
|
path.resolve().relative_to(self.repository_path)
|
|
251
273
|
except (ValueError, OSError):
|
|
252
274
|
continue
|
|
275
|
+
|
|
253
276
|
content_hash, content_text = _read_and_hash_file(path)
|
|
254
277
|
language = LANGUAGE_BY_SUFFIX.get(path.suffix.lower())
|
|
255
278
|
folder_role = self._classify_folder(relative_path)
|
|
256
279
|
is_manifest = path.name in MANIFEST_NAMES
|
|
257
|
-
|
|
280
|
+
|
|
281
|
+
yield FileObservation(
|
|
258
282
|
path=path,
|
|
259
283
|
relative_path=relative_path,
|
|
260
284
|
size_bytes=stat.st_size,
|
|
@@ -264,23 +288,6 @@ class RepositoryScanner:
|
|
|
264
288
|
content=content_text,
|
|
265
289
|
is_manifest=is_manifest,
|
|
266
290
|
)
|
|
267
|
-
files.append(observation)
|
|
268
|
-
if is_manifest:
|
|
269
|
-
manifests.append(self._read_manifest(path, relative_path))
|
|
270
|
-
|
|
271
|
-
language_counts: dict[str, int] = {}
|
|
272
|
-
folder_counts: dict[str, int] = {}
|
|
273
|
-
for file in files:
|
|
274
|
-
if file.language:
|
|
275
|
-
language_counts[file.language] = language_counts.get(file.language, 0) + 1
|
|
276
|
-
folder_counts[file.folder_role.value] = folder_counts.get(file.folder_role.value, 0) + 1
|
|
277
|
-
return RepositoryScan(
|
|
278
|
-
repository_path=self.repository_path,
|
|
279
|
-
files=tuple(files),
|
|
280
|
-
manifests=tuple(manifests),
|
|
281
|
-
language_counts=dict(sorted(language_counts.items())),
|
|
282
|
-
folder_counts=dict(sorted(folder_counts.items())),
|
|
283
|
-
)
|
|
284
291
|
|
|
285
292
|
def _excluded(self, path: Path) -> bool:
|
|
286
293
|
try:
|
|
@@ -289,6 +296,8 @@ class RepositoryScanner:
|
|
|
289
296
|
return True
|
|
290
297
|
if any(part in self.excludes for part in parts):
|
|
291
298
|
return True
|
|
299
|
+
if self.synapignore and self.synapignore.matches(path, is_dir=path.is_dir()):
|
|
300
|
+
return True
|
|
292
301
|
if self.gitignore and self.gitignore.matches(path, is_dir=path.is_dir()):
|
|
293
302
|
return True
|
|
294
303
|
return False
|
|
@@ -55,28 +55,38 @@ class CodeParserRegistry:
|
|
|
55
55
|
return self._parsers[lang_name]
|
|
56
56
|
|
|
57
57
|
def parse(self, path: Path, *, relative_path: str, text: str | None = None) -> CodeParseResult:
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
text
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
58
|
+
try:
|
|
59
|
+
suffix = path.suffix.lower()
|
|
60
|
+
if text is None:
|
|
61
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
62
|
+
|
|
63
|
+
if suffix in PYTHON_SUFFIXES:
|
|
64
|
+
return self._parse_tree_sitter(text, "python", relative_path)
|
|
65
|
+
if suffix in JAVASCRIPT_SUFFIXES:
|
|
66
|
+
return self._parse_tree_sitter(text, "javascript", relative_path)
|
|
67
|
+
if suffix in TYPESCRIPT_SUFFIXES:
|
|
68
|
+
return self._parse_tree_sitter(text, "tsx", relative_path)
|
|
69
|
+
if suffix in GO_SUFFIXES:
|
|
70
|
+
return self._parse_tree_sitter(text, "go", relative_path)
|
|
71
|
+
if suffix in RUST_SUFFIXES:
|
|
72
|
+
return self._parse_tree_sitter(text, "rust", relative_path)
|
|
73
|
+
if suffix in JAVA_SUFFIXES:
|
|
74
|
+
return self._parse_tree_sitter(text, "java", relative_path)
|
|
75
|
+
if suffix in CPP_SUFFIXES:
|
|
76
|
+
return self._parse_tree_sitter(text, "cpp", relative_path)
|
|
77
|
+
if suffix in RUBY_SUFFIXES:
|
|
78
|
+
return self._parse_tree_sitter(text, "ruby", relative_path)
|
|
79
|
+
|
|
80
|
+
return CodeParseResult(path=relative_path, language="unknown", symbols=())
|
|
81
|
+
except Exception as e:
|
|
82
|
+
import structlog
|
|
83
|
+
|
|
84
|
+
structlog.get_logger(__name__).warning(
|
|
85
|
+
"parser_failed", path=relative_path, error=str(e)
|
|
86
|
+
)
|
|
87
|
+
return CodeParseResult(
|
|
88
|
+
path=relative_path, language="unknown", symbols=(), syntax_error=str(e)
|
|
89
|
+
)
|
|
80
90
|
|
|
81
91
|
def _parse_tree_sitter(self, text: str, lang_name: str, relative_path: str) -> CodeParseResult:
|
|
82
92
|
parser = self._get_parser(lang_name)
|
|
@@ -1,20 +1,46 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import
|
|
5
|
-
from collections.abc import Iterator
|
|
3
|
+
import time
|
|
4
|
+
from collections.abc import Callable, Iterator
|
|
6
5
|
from typing import Any
|
|
7
6
|
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
8
9
|
from synap_git.provider.base import LLMProvider, LLMResponse
|
|
9
10
|
|
|
10
11
|
|
|
12
|
+
def _with_retries(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
13
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
14
|
+
attempts = 3
|
|
15
|
+
backoff = 1.0
|
|
16
|
+
for i in range(attempts):
|
|
17
|
+
try:
|
|
18
|
+
return func(*args, **kwargs)
|
|
19
|
+
except httpx.HTTPStatusError as e:
|
|
20
|
+
if e.response.status_code in (429, 500, 502, 503, 504) and i < attempts - 1:
|
|
21
|
+
time.sleep(backoff)
|
|
22
|
+
backoff *= 2.0
|
|
23
|
+
continue
|
|
24
|
+
raise RuntimeError(f"API request failed: {e}") from e
|
|
25
|
+
except httpx.RequestError as e:
|
|
26
|
+
if i < attempts - 1:
|
|
27
|
+
time.sleep(backoff)
|
|
28
|
+
backoff *= 2.0
|
|
29
|
+
continue
|
|
30
|
+
raise RuntimeError(f"API request failed: {e}") from e
|
|
31
|
+
|
|
32
|
+
return wrapper
|
|
33
|
+
|
|
34
|
+
|
|
11
35
|
class GeminiProvider(LLMProvider):
|
|
12
36
|
"""Gemini API provider implementation."""
|
|
13
37
|
|
|
14
38
|
def __init__(self, api_key: str, default_model: str = "gemini-1.5-flash") -> None:
|
|
15
39
|
self.api_key = api_key
|
|
16
40
|
self.default_model = default_model
|
|
41
|
+
self.client = httpx.Client(timeout=30.0)
|
|
17
42
|
|
|
43
|
+
@_with_retries
|
|
18
44
|
def generate(
|
|
19
45
|
self,
|
|
20
46
|
system_prompt: str,
|
|
@@ -25,47 +51,32 @@ class GeminiProvider(LLMProvider):
|
|
|
25
51
|
temperature: float = 0.2,
|
|
26
52
|
) -> LLMResponse:
|
|
27
53
|
model_name = model or self.default_model
|
|
28
|
-
# Strip models/ prefix if present
|
|
29
54
|
if model_name.startswith("models/"):
|
|
30
55
|
model_name = model_name[7:]
|
|
31
56
|
url = f"https://generativelanguage.googleapis.com/v1beta/models/{model_name}:generateContent?key={self.api_key}"
|
|
32
|
-
headers = {
|
|
33
|
-
"Content-Type": "application/json",
|
|
34
|
-
}
|
|
57
|
+
headers = {"Content-Type": "application/json"}
|
|
35
58
|
payload: dict[str, Any] = {
|
|
36
|
-
"contents": [
|
|
37
|
-
{
|
|
38
|
-
"role": "user",
|
|
39
|
-
"parts": [{"text": user_prompt}],
|
|
40
|
-
}
|
|
41
|
-
],
|
|
59
|
+
"contents": [{"role": "user", "parts": [{"text": user_prompt}]}],
|
|
42
60
|
"systemInstruction": {"parts": [{"text": system_prompt}]},
|
|
43
|
-
"generationConfig": {
|
|
44
|
-
"temperature": temperature,
|
|
45
|
-
},
|
|
61
|
+
"generationConfig": {"temperature": temperature},
|
|
46
62
|
}
|
|
47
63
|
if max_tokens:
|
|
48
64
|
payload["generationConfig"]["maxOutputTokens"] = max_tokens
|
|
49
65
|
|
|
50
|
-
|
|
51
|
-
|
|
66
|
+
resp = self.client.post(url, json=payload, headers=headers)
|
|
67
|
+
resp.raise_for_status()
|
|
68
|
+
data = resp.json()
|
|
69
|
+
|
|
70
|
+
candidates = data.get("candidates", [])
|
|
71
|
+
if not candidates:
|
|
72
|
+
raise RuntimeError(f"Gemini API returned no candidates: {data}")
|
|
73
|
+
text_val = candidates[0]["content"]["parts"][0]["text"]
|
|
74
|
+
usage = data.get("usageMetadata", {})
|
|
75
|
+
return LLMResponse(
|
|
76
|
+
content=str(text_val),
|
|
77
|
+
prompt_tokens=int(usage.get("promptTokenCount", 0)),
|
|
78
|
+
completion_tokens=int(usage.get("candidatesTokenCount", 0)),
|
|
52
79
|
)
|
|
53
|
-
try:
|
|
54
|
-
with urllib.request.urlopen(req, timeout=30.0) as resp: # nosec B310
|
|
55
|
-
data = json.loads(resp.read().decode("utf-8"))
|
|
56
|
-
candidates = data.get("candidates", [])
|
|
57
|
-
if not candidates:
|
|
58
|
-
raise RuntimeError(f"Gemini API returned no candidates: {data}")
|
|
59
|
-
text_val = candidates[0]["content"]["parts"][0]["text"]
|
|
60
|
-
# Gemini does not always provide exact token counts in basic responses, or they might be in usageMetadata
|
|
61
|
-
usage = data.get("usageMetadata", {})
|
|
62
|
-
return LLMResponse(
|
|
63
|
-
content=str(text_val),
|
|
64
|
-
prompt_tokens=int(usage.get("promptTokenCount", 0)),
|
|
65
|
-
completion_tokens=int(usage.get("candidatesTokenCount", 0)),
|
|
66
|
-
)
|
|
67
|
-
except Exception as exc:
|
|
68
|
-
raise RuntimeError(f"Gemini generate failed: {exc}") from exc
|
|
69
80
|
|
|
70
81
|
def generate_stream(
|
|
71
82
|
self,
|
|
@@ -81,6 +92,7 @@ class GeminiProvider(LLMProvider):
|
|
|
81
92
|
)
|
|
82
93
|
yield res.content
|
|
83
94
|
|
|
95
|
+
@_with_retries
|
|
84
96
|
def embed(
|
|
85
97
|
self,
|
|
86
98
|
text: str,
|
|
@@ -91,20 +103,14 @@ class GeminiProvider(LLMProvider):
|
|
|
91
103
|
if not model_name.startswith("models/"):
|
|
92
104
|
model_name = f"models/{model_name}"
|
|
93
105
|
url = f"https://generativelanguage.googleapis.com/v1beta/{model_name}:embedContent?key={self.api_key}"
|
|
94
|
-
headers = {
|
|
95
|
-
"Content-Type": "application/json",
|
|
96
|
-
}
|
|
106
|
+
headers = {"Content-Type": "application/json"}
|
|
97
107
|
payload = {"model": model_name, "content": {"parts": [{"text": text}]}}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
embedding = data["embedding"]["values"]
|
|
105
|
-
return [float(x) for x in embedding]
|
|
106
|
-
except Exception as exc:
|
|
107
|
-
raise RuntimeError(f"Gemini embed failed: {exc}") from exc
|
|
108
|
+
|
|
109
|
+
resp = self.client.post(url, json=payload, headers=headers)
|
|
110
|
+
resp.raise_for_status()
|
|
111
|
+
data = resp.json()
|
|
112
|
+
embedding = data["embedding"]["values"]
|
|
113
|
+
return [float(x) for x in embedding]
|
|
108
114
|
|
|
109
115
|
def count_tokens(self, text: str) -> int:
|
|
110
116
|
import tiktoken
|