synap-git 1.2.3__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {synap_git-1.2.3 → synap_git-2.0.0}/CHANGELOG.md +11 -0
  2. {synap_git-1.2.3 → synap_git-2.0.0}/PKG-INFO +5 -5
  3. {synap_git-1.2.3 → synap_git-2.0.0}/README.md +4 -4
  4. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/__init__.py +1 -1
  5. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/engine.py +123 -22
  6. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/scanner.py +36 -27
  7. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/parser/registry.py +32 -22
  8. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/gemini.py +53 -47
  9. synap_git-2.0.0/src/synap_git/provider/ollama.py +119 -0
  10. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/retrieval/engine.py +144 -44
  11. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/storage/sqlite.py +114 -13
  12. synap_git-1.2.3/src/synap_git/provider/ollama.py +0 -140
  13. {synap_git-1.2.3 → synap_git-2.0.0}/.gitignore +0 -0
  14. {synap_git-1.2.3 → synap_git-2.0.0}/.synap.example/README.md +0 -0
  15. {synap_git-1.2.3 → synap_git-2.0.0}/LICENSE.md +0 -0
  16. {synap_git-1.2.3 → synap_git-2.0.0}/pyproject.toml +0 -0
  17. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/api/__init__.py +0 -0
  18. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/api/app.py +0 -0
  19. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/api/static/index.html +0 -0
  20. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/cli/__init__.py +0 -0
  21. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/cli/__main__.py +0 -0
  22. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/cli/main.py +0 -0
  23. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/config.py +0 -0
  24. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/diagnostics/__init__.py +0 -0
  25. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/diagnostics/logger.py +0 -0
  26. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/diagnostics/tracing.py +0 -0
  27. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/embeddings/__init__.py +0 -0
  28. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/git/__init__.py +0 -0
  29. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/git/state.py +0 -0
  30. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/__init__.py +0 -0
  31. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/daemon.py +0 -0
  32. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/indexer/wiki.py +0 -0
  33. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/mcp/__init__.py +0 -0
  34. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/mcp/server.py +0 -0
  35. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/parser/__init__.py +0 -0
  36. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/anthropic.py +0 -0
  37. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/base.py +0 -0
  38. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/factory.py +0 -0
  39. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/openai.py +0 -0
  40. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/provider/openrouter.py +0 -0
  41. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/py.typed +0 -0
  42. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/retrieval/__init__.py +0 -0
  43. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/retrieval/memory.py +0 -0
  44. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/storage/__init__.py +0 -0
  45. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/utils/__init__.py +0 -0
  46. {synap_git-1.2.3 → synap_git-2.0.0}/src/synap_git/utils/serialization.py +0 -0
@@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [2.0.0] - 2026-05-29
9
+
10
+ ### Added
11
+ - **.synapignore Support**: Implemented context-aware filtering to prevent traversing symlinks, binary files, and files listed in `.synapignore` or `.gitignore`.
12
+ - **FTS5 Integration**: Shifted semantic full-text search directly to SQLite FTS5 with bounding limits (`LIMIT 50`) for `O(1)` lexical retrieval scaling on 10,000+ file monorepos.
13
+
14
+ ### Fixed
15
+ - **LLM Hangs in Test Suite**: Isolated testing environment by strictly bypassing real LLM provider configs (`SYNAP_LLM_PROVIDER=""`) preventing test CI suite hangs.
16
+ - **Repository Path Scoping in Hybrid Engine**: Fixed a bug where `repo_path` initialization mistakenly truncated the path to `/tmp` in tests, failing contextual snippet aggregation.
17
+ - **Unbounded BM25 Expansion**: Bounded full-text search token extraction and BM25 results, preventing out-of-memory cascading in CTE traversal during monorepo indexing.
18
+
8
19
  ## [1.2.3] - 2026-05-28
9
20
 
10
21
  ### Added
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: synap-git
3
- Version: 1.2.3
3
+ Version: 2.0.0
4
4
  Summary: Persistent structural context infrastructure for AI coding agents.
5
5
  Project-URL: Homepage, https://github.com/saahilpal/synap-git
6
6
  Project-URL: Repository, https://github.com/saahilpal/synap-git
@@ -52,10 +52,10 @@ Description-Content-Type: text/markdown
52
52
 
53
53
  Local Git-aware structural context engine for AI coding agents.
54
54
 
55
- [![CI Status](https://github.com/saahilpal/synap-git/actions/workflows/ci.yml/badge.svg)](https://github.com/saahilpal/synap-git/actions)
56
- [![Version](https://img.shields.io/pypi/v/synap-git?color=3b82f6)](https://img.shields.io/project/synap-git/)
57
- [![Python Version](https://img.shields.io/pypi/pyversions/synap-git)](https://img.shields.io/project/synap-git/)
58
- [![License](https://img.shields.io/github/license/saahilpal/synap-git?color=cbd5e1)](LICENSE.md)
55
+ [![CI Status](https://github.com/saahilpal/synapse/actions/workflows/ci.yml/badge.svg)](https://github.com/saahilpal/synapse/actions)
56
+ [![Version](https://img.shields.io/pypi/v/synap-git?color=3b82f6)](https://pypi.org/project/synap-git/)
57
+ [![Python Version](https://img.shields.io/pypi/pyversions/synap-git)](https://pypi.org/project/synap-git/)
58
+ [![License](https://img.shields.io/github/license/saahilpal/synapse?color=cbd5e1)](LICENSE.md)
59
59
 
60
60
  ## The Problem
61
61
 
@@ -2,10 +2,10 @@
2
2
 
3
3
  Local Git-aware structural context engine for AI coding agents.
4
4
 
5
- [![CI Status](https://github.com/saahilpal/synap-git/actions/workflows/ci.yml/badge.svg)](https://github.com/saahilpal/synap-git/actions)
6
- [![Version](https://img.shields.io/pypi/v/synap-git?color=3b82f6)](https://img.shields.io/project/synap-git/)
7
- [![Python Version](https://img.shields.io/pypi/pyversions/synap-git)](https://img.shields.io/project/synap-git/)
8
- [![License](https://img.shields.io/github/license/saahilpal/synap-git?color=cbd5e1)](LICENSE.md)
5
+ [![CI Status](https://github.com/saahilpal/synapse/actions/workflows/ci.yml/badge.svg)](https://github.com/saahilpal/synapse/actions)
6
+ [![Version](https://img.shields.io/pypi/v/synap-git?color=3b82f6)](https://pypi.org/project/synap-git/)
7
+ [![Python Version](https://img.shields.io/pypi/pyversions/synap-git)](https://pypi.org/project/synap-git/)
8
+ [![License](https://img.shields.io/github/license/saahilpal/synapse?color=cbd5e1)](LICENSE.md)
9
9
 
10
10
  ## The Problem
11
11
 
@@ -2,4 +2,4 @@
2
2
 
3
3
  __all__ = ["__version__"]
4
4
 
5
- __version__ = "1.2.3"
5
+ __version__ = "2.0.0"
@@ -69,6 +69,7 @@ class SynapRuntime:
69
69
  self.parser_registry = CodeParserRegistry()
70
70
  self.llm_provider = get_llm_provider(settings)
71
71
  self.retrieval_engine = HybridRetrievalEngine(
72
+ repo_path=self.settings.repository_path,
72
73
  store=self.store,
73
74
  llm_provider=self.llm_provider,
74
75
  trace_store=self.trace_store,
@@ -244,6 +245,44 @@ class SynapRuntime:
244
245
 
245
246
  return commit
246
247
 
248
+ def _schedule_embedding(
249
+ self, file_id: str, symbols: list[dict[str, Any]], content_hash: str
250
+ ) -> None:
251
+ provider = self.llm_provider
252
+ if not provider:
253
+ return
254
+
255
+ def _worker() -> None:
256
+ import structlog
257
+
258
+ logger = structlog.get_logger()
259
+ try:
260
+ batches = [symbols[i : i + 50] for i in range(0, len(symbols), 50)]
261
+ model_name = getattr(provider, "default_model", "unknown")
262
+ for batch in batches:
263
+ for sym in batch:
264
+ try:
265
+ # Avoid re-embedding if unchanged?
266
+ text = sym["name"] + " " + sym.get("metadata", {}).get("docstring", "")
267
+ vector = provider.embed(text)
268
+ self.store.put_embedding(
269
+ sym["symbol_id"], model_name, "1.0", "1.0", vector, content_hash
270
+ )
271
+ except Exception as e:
272
+ logger.warning("background_embed_failed_for_symbol", error=str(e))
273
+ except Exception as e:
274
+ logger.warning("background_embed_failed", error=str(e))
275
+
276
+ try:
277
+ import asyncio
278
+
279
+ loop = asyncio.get_running_loop()
280
+ loop.create_task(asyncio.to_thread(_worker))
281
+ except RuntimeError:
282
+ import threading
283
+
284
+ threading.Thread(target=_worker, daemon=True).start()
285
+
247
286
  def _first_run_index(self, git_state: GitState) -> str | None:
248
287
  self.logger.info("first_run_indexing_started", branch=git_state.effective_branch)
249
288
  import time
@@ -254,16 +293,9 @@ class SynapRuntime:
254
293
  repository_path=self.settings.repository_path,
255
294
  max_file_bytes=self.settings.max_file_bytes,
256
295
  )
257
- scan = scanner.scan()
258
-
259
- files_to_parse = []
260
- for file_info in scan.files:
261
- files_to_parse.append((file_info.path, file_info.relative_path, file_info.content))
262
-
263
- num_files = len(files_to_parse)
296
+ num_files = scanner.count_files()
264
297
  chunk_size = 500
265
298
  parsed_results = []
266
-
267
299
  # Parallel parsing using ProcessPoolExecutor
268
300
  import multiprocessing
269
301
  from concurrent.futures import ProcessPoolExecutor
@@ -301,16 +333,22 @@ class SynapRuntime:
301
333
  task = progress.add_task("[yellow]Parsing codebase...[/yellow]", total=num_files)
302
334
 
303
335
  with ProcessPoolExecutor(max_workers=num_workers) as executor:
304
- for chunk_idx in range(0, num_files, chunk_size):
305
- chunk = files_to_parse[chunk_idx : chunk_idx + chunk_size]
336
+ from itertools import islice
337
+
338
+ file_iterator = scanner.scan()
339
+ while True:
340
+ chunk_files = list(islice(file_iterator, chunk_size))
341
+ if not chunk_files:
342
+ break
343
+
344
+ file_info_map = {f.relative_path: f for f in chunk_files}
345
+ chunk = [(f.path, f.relative_path, f.content) for f in chunk_files]
306
346
  futures = [executor.submit(_parse_worker, item) for item in chunk]
307
347
  chunk_results = [f.result() for f in futures]
308
348
 
309
349
  for res in chunk_results:
310
350
  rel_path = res["path"]
311
- matching_info = next(
312
- (f for f in scan.files if f.relative_path == rel_path), None
313
- )
351
+ matching_info = file_info_map.get(rel_path)
314
352
  git_oid = matching_info.git_oid if matching_info else ""
315
353
  content_hash = matching_info.content_hash if matching_info else ""
316
354
 
@@ -328,6 +366,7 @@ class SynapRuntime:
328
366
  language=res["language"],
329
367
  symbols=res["symbols"],
330
368
  )
369
+ self._schedule_embedding(file_id_hash, res["symbols"], content_hash)
331
370
  self.store.enqueue_wiki(rel_path)
332
371
  parsed_results.append((file_id_hash, rel_path, res["imports"]))
333
372
 
@@ -336,20 +375,26 @@ class SynapRuntime:
336
375
  self._resolve_and_insert_edges(parsed_results)
337
376
  parsed_results.clear()
338
377
 
339
- progress.advance(task, len(chunk))
378
+ progress.advance(task, len(chunk_files))
340
379
  del chunk_results
341
380
  else:
342
381
  with ProcessPoolExecutor(max_workers=num_workers) as executor:
343
- for chunk_idx in range(0, num_files, chunk_size):
344
- chunk = files_to_parse[chunk_idx : chunk_idx + chunk_size]
382
+ from itertools import islice
383
+
384
+ file_iterator = scanner.scan()
385
+ while True:
386
+ chunk_files = list(islice(file_iterator, chunk_size))
387
+ if not chunk_files:
388
+ break
389
+
390
+ file_info_map = {f.relative_path: f for f in chunk_files}
391
+ chunk = [(f.path, f.relative_path, f.content) for f in chunk_files]
345
392
  futures = [executor.submit(_parse_worker, item) for item in chunk]
346
393
  chunk_results = [f.result() for f in futures]
347
394
 
348
395
  for res in chunk_results:
349
396
  rel_path = res["path"]
350
- matching_info = next(
351
- (f for f in scan.files if f.relative_path == rel_path), None
352
- )
397
+ matching_info = file_info_map.get(rel_path)
353
398
  git_oid = matching_info.git_oid if matching_info else ""
354
399
  content_hash = matching_info.content_hash if matching_info else ""
355
400
 
@@ -367,6 +412,7 @@ class SynapRuntime:
367
412
  language=res["language"],
368
413
  symbols=res["symbols"],
369
414
  )
415
+ self._schedule_embedding(file_id_hash, res["symbols"], content_hash)
370
416
  self.store.enqueue_wiki(rel_path)
371
417
  parsed_results.append((file_id_hash, rel_path, res["imports"]))
372
418
 
@@ -426,6 +472,7 @@ class SynapRuntime:
426
472
 
427
473
  added_or_modified = []
428
474
  deleted = []
475
+ renames = []
429
476
  for line in res.stdout.splitlines():
430
477
  if not line.strip():
431
478
  continue
@@ -433,13 +480,41 @@ class SynapRuntime:
433
480
  if len(parts) >= 2:
434
481
  status = parts[0]
435
482
  if status.startswith("R"):
436
- deleted.append(parts[1])
483
+ renames.append((parts[1], parts[2]))
437
484
  added_or_modified.append(parts[2])
438
485
  elif status == "D":
439
486
  deleted.append(parts[1])
440
487
  else:
441
488
  added_or_modified.append(parts[1])
442
489
 
490
+ import hashlib
491
+
492
+ # Apply renames to preserve identity
493
+ with self.store.connect() as conn:
494
+ for old_path, new_path in renames:
495
+ file_row = conn.execute(
496
+ "SELECT file_id, content_hash FROM files WHERE path = ?", (old_path,)
497
+ ).fetchone()
498
+ if file_row:
499
+ old_file_id = file_row["file_id"]
500
+ content_hash = file_row["content_hash"]
501
+ new_file_id = hashlib.sha256(
502
+ (new_path + content_hash).encode("utf-8")
503
+ ).hexdigest()
504
+
505
+ conn.execute("PRAGMA foreign_keys=OFF")
506
+ try:
507
+ conn.execute(
508
+ "UPDATE files SET file_id = ?, path = ? WHERE file_id = ?",
509
+ (new_file_id, new_path, old_file_id),
510
+ )
511
+ conn.execute(
512
+ "UPDATE symbols SET file_id = ? WHERE file_id = ?",
513
+ (new_file_id, old_file_id),
514
+ )
515
+ finally:
516
+ conn.execute("PRAGMA foreign_keys=ON")
517
+
443
518
  # Filter added/modified files
444
519
  scanner = RepositoryScanner(
445
520
  repository_path=self.settings.repository_path,
@@ -549,6 +624,7 @@ class SynapRuntime:
549
624
  language=parse_result.language,
550
625
  symbols=symbols_list,
551
626
  )
627
+ self._schedule_embedding(file_id_hash, symbols_list, content_hash)
552
628
 
553
629
  self.store.set_wiki_status(rel_path, None, "stale")
554
630
  self.store.enqueue_wiki(rel_path)
@@ -701,11 +777,23 @@ class SynapRuntime:
701
777
  for row in rows:
702
778
  fts_symbols_by_name.setdefault(row["name"], []).append(dict(row))
703
779
 
780
+ target_symbols_dict: dict[str, dict[str, list[dict[str, Any]]]] = {}
781
+ for t_file_id, syms in target_symbols_by_file.items():
782
+ name_map: dict[str, Any] = {}
783
+ for sym in syms:
784
+ name_map.setdefault(sym["name"], []).append(sym)
785
+ target_symbols_dict[t_file_id] = name_map
786
+
704
787
  for file_id, rel_path, imports in parsed_results:
705
788
  file_symbols = file_symbols_map.get(file_id, [])
706
789
  current_path = Path(rel_path)
790
+ edges_added_for_file = 0
791
+ max_edges_per_file = 1000
707
792
 
708
793
  for imp in imports:
794
+ if edges_added_for_file >= max_edges_per_file:
795
+ break
796
+
709
797
  target_file_id = None
710
798
  if ":" in imp:
711
799
  module_part, target_name = imp.split(":", 1)
@@ -728,15 +816,27 @@ class SynapRuntime:
728
816
  else:
729
817
  target_file_id = module_key_to_file_id.get(module_part)
730
818
 
819
+ # Fuzzy fallback
820
+ if not target_file_id:
821
+ for mk, fid in module_key_to_file_id.items():
822
+ if mk.endswith(module_part) or mk.endswith(f".{module_part}"):
823
+ target_file_id = fid
824
+ break
825
+
731
826
  target_symbols = []
732
827
  if target_file_id:
733
- all_target_syms = target_symbols_by_file.get(target_file_id, [])
734
- target_symbols = [ts for ts in all_target_syms if ts["name"] == target_name]
828
+ target_symbols = target_symbols_dict.get(target_file_id, {}).get(
829
+ target_name, []
830
+ )
735
831
  else:
736
832
  target_symbols = fts_symbols_by_name.get(target_name, [])
737
833
 
738
834
  for ts in target_symbols:
835
+ if edges_added_for_file >= max_edges_per_file:
836
+ break
739
837
  for fs in file_symbols:
838
+ if edges_added_for_file >= max_edges_per_file:
839
+ break
740
840
  if fs["symbol_id"] == ts["symbol_id"]:
741
841
  continue
742
842
  edge_id = stable_hash(
@@ -747,6 +847,7 @@ class SynapRuntime:
747
847
  }
748
848
  )
749
849
  edges_to_insert.append((edge_id, fs["symbol_id"], ts["symbol_id"]))
850
+ edges_added_for_file += 1
750
851
 
751
852
  if edges_to_insert:
752
853
  conn.executemany(
@@ -1,11 +1,11 @@
1
1
  from __future__ import annotations
2
2
 
3
- import asyncio
4
3
  import hashlib
5
4
  import json
6
5
  import re
7
6
  import subprocess
8
7
  import tomllib
8
+ from collections.abc import Iterator
9
9
  from dataclasses import dataclass, field
10
10
  from enum import StrEnum
11
11
  from pathlib import Path
@@ -196,6 +196,7 @@ class RepositoryScanner:
196
196
  self.excludes = excludes or DEFAULT_EXCLUDES
197
197
  self.max_file_bytes = max_file_bytes
198
198
  self.gitignore = None
199
+ self.synapignore = None
199
200
 
200
201
  gitignore_path = self.repository_path / ".gitignore"
201
202
  if gitignore_path.exists():
@@ -205,13 +206,15 @@ class RepositoryScanner:
205
206
  except Exception as e:
206
207
  logger.warning("failed_to_load_gitignore", error=str(e))
207
208
 
208
- async def scan_async(self) -> RepositoryScan:
209
- return await asyncio.to_thread(self.scan)
210
-
211
- def scan(self) -> RepositoryScan:
212
- files: list[FileObservation] = []
213
- manifests: list[ManifestObservation] = []
209
+ synapignore_path = self.repository_path / ".synapignore"
210
+ if synapignore_path.exists():
211
+ try:
212
+ lines = synapignore_path.read_text(encoding="utf-8").splitlines()
213
+ self.synapignore = GitIgnoreSpec(self.repository_path, lines)
214
+ except Exception as e:
215
+ logger.warning("failed_to_load_synapignore", error=str(e))
214
216
 
217
+ def _get_paths(self) -> list[Path]:
215
218
  paths: list[Path] = []
216
219
  try:
217
220
  result = subprocess.run( # noqa: S603
@@ -227,10 +230,29 @@ class RepositoryScanner:
227
230
  except UnicodeDecodeError:
228
231
  pass
229
232
  except (subprocess.CalledProcessError, FileNotFoundError):
230
- # Fallback to rglob if not a git repository
231
233
  paths = list(self.repository_path.rglob("*"))
234
+ return paths
235
+
236
+ def count_files(self) -> int:
237
+ count = 0
238
+ for path in self._get_paths():
239
+ if not path.is_file():
240
+ continue
241
+ if self._excluded(path):
242
+ continue
243
+ try:
244
+ stat = path.stat()
245
+ if stat.st_size > self.max_file_bytes:
246
+ continue
247
+ except OSError:
248
+ continue
249
+ if _is_binary_file(path):
250
+ continue
251
+ count += 1
252
+ return count
232
253
 
233
- for path in paths:
254
+ def scan(self) -> Iterator[FileObservation]:
255
+ for path in self._get_paths():
234
256
  if not path.is_file():
235
257
  continue
236
258
  if self._excluded(path):
@@ -250,11 +272,13 @@ class RepositoryScanner:
250
272
  path.resolve().relative_to(self.repository_path)
251
273
  except (ValueError, OSError):
252
274
  continue
275
+
253
276
  content_hash, content_text = _read_and_hash_file(path)
254
277
  language = LANGUAGE_BY_SUFFIX.get(path.suffix.lower())
255
278
  folder_role = self._classify_folder(relative_path)
256
279
  is_manifest = path.name in MANIFEST_NAMES
257
- observation = FileObservation(
280
+
281
+ yield FileObservation(
258
282
  path=path,
259
283
  relative_path=relative_path,
260
284
  size_bytes=stat.st_size,
@@ -264,23 +288,6 @@ class RepositoryScanner:
264
288
  content=content_text,
265
289
  is_manifest=is_manifest,
266
290
  )
267
- files.append(observation)
268
- if is_manifest:
269
- manifests.append(self._read_manifest(path, relative_path))
270
-
271
- language_counts: dict[str, int] = {}
272
- folder_counts: dict[str, int] = {}
273
- for file in files:
274
- if file.language:
275
- language_counts[file.language] = language_counts.get(file.language, 0) + 1
276
- folder_counts[file.folder_role.value] = folder_counts.get(file.folder_role.value, 0) + 1
277
- return RepositoryScan(
278
- repository_path=self.repository_path,
279
- files=tuple(files),
280
- manifests=tuple(manifests),
281
- language_counts=dict(sorted(language_counts.items())),
282
- folder_counts=dict(sorted(folder_counts.items())),
283
- )
284
291
 
285
292
  def _excluded(self, path: Path) -> bool:
286
293
  try:
@@ -289,6 +296,8 @@ class RepositoryScanner:
289
296
  return True
290
297
  if any(part in self.excludes for part in parts):
291
298
  return True
299
+ if self.synapignore and self.synapignore.matches(path, is_dir=path.is_dir()):
300
+ return True
292
301
  if self.gitignore and self.gitignore.matches(path, is_dir=path.is_dir()):
293
302
  return True
294
303
  return False
@@ -55,28 +55,38 @@ class CodeParserRegistry:
55
55
  return self._parsers[lang_name]
56
56
 
57
57
  def parse(self, path: Path, *, relative_path: str, text: str | None = None) -> CodeParseResult:
58
- suffix = path.suffix.lower()
59
- if text is None:
60
- text = path.read_text(encoding="utf-8", errors="replace")
61
-
62
- if suffix in PYTHON_SUFFIXES:
63
- return self._parse_tree_sitter(text, "python", relative_path)
64
- if suffix in JAVASCRIPT_SUFFIXES:
65
- return self._parse_tree_sitter(text, "javascript", relative_path)
66
- if suffix in TYPESCRIPT_SUFFIXES:
67
- return self._parse_tree_sitter(text, "tsx", relative_path)
68
- if suffix in GO_SUFFIXES:
69
- return self._parse_tree_sitter(text, "go", relative_path)
70
- if suffix in RUST_SUFFIXES:
71
- return self._parse_tree_sitter(text, "rust", relative_path)
72
- if suffix in JAVA_SUFFIXES:
73
- return self._parse_tree_sitter(text, "java", relative_path)
74
- if suffix in CPP_SUFFIXES:
75
- return self._parse_tree_sitter(text, "cpp", relative_path)
76
- if suffix in RUBY_SUFFIXES:
77
- return self._parse_tree_sitter(text, "ruby", relative_path)
78
-
79
- return CodeParseResult(path=relative_path, language="unknown", symbols=())
58
+ try:
59
+ suffix = path.suffix.lower()
60
+ if text is None:
61
+ text = path.read_text(encoding="utf-8", errors="replace")
62
+
63
+ if suffix in PYTHON_SUFFIXES:
64
+ return self._parse_tree_sitter(text, "python", relative_path)
65
+ if suffix in JAVASCRIPT_SUFFIXES:
66
+ return self._parse_tree_sitter(text, "javascript", relative_path)
67
+ if suffix in TYPESCRIPT_SUFFIXES:
68
+ return self._parse_tree_sitter(text, "tsx", relative_path)
69
+ if suffix in GO_SUFFIXES:
70
+ return self._parse_tree_sitter(text, "go", relative_path)
71
+ if suffix in RUST_SUFFIXES:
72
+ return self._parse_tree_sitter(text, "rust", relative_path)
73
+ if suffix in JAVA_SUFFIXES:
74
+ return self._parse_tree_sitter(text, "java", relative_path)
75
+ if suffix in CPP_SUFFIXES:
76
+ return self._parse_tree_sitter(text, "cpp", relative_path)
77
+ if suffix in RUBY_SUFFIXES:
78
+ return self._parse_tree_sitter(text, "ruby", relative_path)
79
+
80
+ return CodeParseResult(path=relative_path, language="unknown", symbols=())
81
+ except Exception as e:
82
+ import structlog
83
+
84
+ structlog.get_logger(__name__).warning(
85
+ "parser_failed", path=relative_path, error=str(e)
86
+ )
87
+ return CodeParseResult(
88
+ path=relative_path, language="unknown", symbols=(), syntax_error=str(e)
89
+ )
80
90
 
81
91
  def _parse_tree_sitter(self, text: str, lang_name: str, relative_path: str) -> CodeParseResult:
82
92
  parser = self._get_parser(lang_name)
@@ -1,20 +1,46 @@
1
1
  from __future__ import annotations
2
2
 
3
- import json
4
- import urllib.request
5
- from collections.abc import Iterator
3
+ import time
4
+ from collections.abc import Callable, Iterator
6
5
  from typing import Any
7
6
 
7
+ import httpx
8
+
8
9
  from synap_git.provider.base import LLMProvider, LLMResponse
9
10
 
10
11
 
12
+ def _with_retries(func: Callable[..., Any]) -> Callable[..., Any]:
13
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
14
+ attempts = 3
15
+ backoff = 1.0
16
+ for i in range(attempts):
17
+ try:
18
+ return func(*args, **kwargs)
19
+ except httpx.HTTPStatusError as e:
20
+ if e.response.status_code in (429, 500, 502, 503, 504) and i < attempts - 1:
21
+ time.sleep(backoff)
22
+ backoff *= 2.0
23
+ continue
24
+ raise RuntimeError(f"API request failed: {e}") from e
25
+ except httpx.RequestError as e:
26
+ if i < attempts - 1:
27
+ time.sleep(backoff)
28
+ backoff *= 2.0
29
+ continue
30
+ raise RuntimeError(f"API request failed: {e}") from e
31
+
32
+ return wrapper
33
+
34
+
11
35
  class GeminiProvider(LLMProvider):
12
36
  """Gemini API provider implementation."""
13
37
 
14
38
  def __init__(self, api_key: str, default_model: str = "gemini-1.5-flash") -> None:
15
39
  self.api_key = api_key
16
40
  self.default_model = default_model
41
+ self.client = httpx.Client(timeout=30.0)
17
42
 
43
+ @_with_retries
18
44
  def generate(
19
45
  self,
20
46
  system_prompt: str,
@@ -25,47 +51,32 @@ class GeminiProvider(LLMProvider):
25
51
  temperature: float = 0.2,
26
52
  ) -> LLMResponse:
27
53
  model_name = model or self.default_model
28
- # Strip models/ prefix if present
29
54
  if model_name.startswith("models/"):
30
55
  model_name = model_name[7:]
31
56
  url = f"https://generativelanguage.googleapis.com/v1beta/models/{model_name}:generateContent?key={self.api_key}"
32
- headers = {
33
- "Content-Type": "application/json",
34
- }
57
+ headers = {"Content-Type": "application/json"}
35
58
  payload: dict[str, Any] = {
36
- "contents": [
37
- {
38
- "role": "user",
39
- "parts": [{"text": user_prompt}],
40
- }
41
- ],
59
+ "contents": [{"role": "user", "parts": [{"text": user_prompt}]}],
42
60
  "systemInstruction": {"parts": [{"text": system_prompt}]},
43
- "generationConfig": {
44
- "temperature": temperature,
45
- },
61
+ "generationConfig": {"temperature": temperature},
46
62
  }
47
63
  if max_tokens:
48
64
  payload["generationConfig"]["maxOutputTokens"] = max_tokens
49
65
 
50
- req = urllib.request.Request(
51
- url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
66
+ resp = self.client.post(url, json=payload, headers=headers)
67
+ resp.raise_for_status()
68
+ data = resp.json()
69
+
70
+ candidates = data.get("candidates", [])
71
+ if not candidates:
72
+ raise RuntimeError(f"Gemini API returned no candidates: {data}")
73
+ text_val = candidates[0]["content"]["parts"][0]["text"]
74
+ usage = data.get("usageMetadata", {})
75
+ return LLMResponse(
76
+ content=str(text_val),
77
+ prompt_tokens=int(usage.get("promptTokenCount", 0)),
78
+ completion_tokens=int(usage.get("candidatesTokenCount", 0)),
52
79
  )
53
- try:
54
- with urllib.request.urlopen(req, timeout=30.0) as resp: # nosec B310
55
- data = json.loads(resp.read().decode("utf-8"))
56
- candidates = data.get("candidates", [])
57
- if not candidates:
58
- raise RuntimeError(f"Gemini API returned no candidates: {data}")
59
- text_val = candidates[0]["content"]["parts"][0]["text"]
60
- # Gemini does not always provide exact token counts in basic responses, or they might be in usageMetadata
61
- usage = data.get("usageMetadata", {})
62
- return LLMResponse(
63
- content=str(text_val),
64
- prompt_tokens=int(usage.get("promptTokenCount", 0)),
65
- completion_tokens=int(usage.get("candidatesTokenCount", 0)),
66
- )
67
- except Exception as exc:
68
- raise RuntimeError(f"Gemini generate failed: {exc}") from exc
69
80
 
70
81
  def generate_stream(
71
82
  self,
@@ -81,6 +92,7 @@ class GeminiProvider(LLMProvider):
81
92
  )
82
93
  yield res.content
83
94
 
95
+ @_with_retries
84
96
  def embed(
85
97
  self,
86
98
  text: str,
@@ -91,20 +103,14 @@ class GeminiProvider(LLMProvider):
91
103
  if not model_name.startswith("models/"):
92
104
  model_name = f"models/{model_name}"
93
105
  url = f"https://generativelanguage.googleapis.com/v1beta/{model_name}:embedContent?key={self.api_key}"
94
- headers = {
95
- "Content-Type": "application/json",
96
- }
106
+ headers = {"Content-Type": "application/json"}
97
107
  payload = {"model": model_name, "content": {"parts": [{"text": text}]}}
98
- req = urllib.request.Request(
99
- url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST"
100
- )
101
- try:
102
- with urllib.request.urlopen(req, timeout=30.0) as resp: # nosec B310
103
- data = json.loads(resp.read().decode("utf-8"))
104
- embedding = data["embedding"]["values"]
105
- return [float(x) for x in embedding]
106
- except Exception as exc:
107
- raise RuntimeError(f"Gemini embed failed: {exc}") from exc
108
+
109
+ resp = self.client.post(url, json=payload, headers=headers)
110
+ resp.raise_for_status()
111
+ data = resp.json()
112
+ embedding = data["embedding"]["values"]
113
+ return [float(x) for x in embedding]
108
114
 
109
115
  def count_tokens(self, text: str) -> int:
110
116
  import tiktoken