synap-git 1.2.3__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {synap_git-1.2.3 → synap_git-2.1.0}/CHANGELOG.md +25 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/PKG-INFO +5 -5
- {synap_git-1.2.3 → synap_git-2.1.0}/README.md +4 -4
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/__init__.py +1 -1
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/api/app.py +4 -2
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/cli/main.py +54 -17
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/config.py +3 -1
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/diagnostics/logger.py +2 -2
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/git/state.py +4 -2
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/indexer/daemon.py +8 -4
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/indexer/engine.py +158 -32
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/indexer/scanner.py +36 -27
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/indexer/wiki.py +8 -4
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/mcp/server.py +12 -4
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/parser/registry.py +34 -24
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/provider/anthropic.py +5 -1
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/provider/gemini.py +53 -47
- synap_git-2.1.0/src/synap_git/provider/ollama.py +119 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/provider/openai.py +5 -1
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/provider/openrouter.py +5 -1
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/retrieval/engine.py +150 -46
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/storage/sqlite.py +123 -17
- synap_git-1.2.3/src/synap_git/provider/ollama.py +0 -140
- {synap_git-1.2.3 → synap_git-2.1.0}/.gitignore +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/.synap.example/README.md +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/LICENSE.md +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/pyproject.toml +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/api/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/api/static/index.html +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/cli/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/cli/__main__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/diagnostics/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/diagnostics/tracing.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/embeddings/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/git/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/indexer/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/mcp/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/parser/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/provider/base.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/provider/factory.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/py.typed +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/retrieval/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/retrieval/memory.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/storage/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/utils/__init__.py +0 -0
- {synap_git-1.2.3 → synap_git-2.1.0}/src/synap_git/utils/serialization.py +0 -0
|
@@ -5,6 +5,31 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [2.1.0] - 2026-05-29
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **CLI `search` command**: New user-facing command for executing hybrid structural searches directly from the terminal.
|
|
12
|
+
- **`embed_provider` configuration**: Added explicit control over the vector embedding provider via `SYNAP_EMBED_PROVIDER` or `config.toml`.
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
- **CRITICAL: Metadata Nullability Bug**: Resolved a type mismatch in `CodeSymbol` where `metadata=None` caused background worker crashes. Enforced strict `dict` contract with `default_factory`.
|
|
16
|
+
- **CRITICAL: Exception Swallowing**: Eradicated 26 instances of silent failure (`except Exception: pass`) across all core subsystems (API, Indexer, Storage, Retrieval, MCP, Providers). Replaced with structured, traceback-aware logging.
|
|
17
|
+
- **CRITICAL: SQLite Schema Integrity**: Fixed `NOT NULL` constraint failure in `llm_calls` table by aligning schema with insertion logic and adding `cost_usd` support.
|
|
18
|
+
- **HIGH: Repository-Local Logging**: Redirected daemon logs from a global shared directory to repository-local `.synap/logs` to prevent cross-repository log corruption and lock contention.
|
|
19
|
+
- **HIGH: Rollback Atomicity**: Reordered `synap rollback` operations to ensure Git state is successfully restored before purging index checkpoints.
|
|
20
|
+
- **MEDIUM: Provider Resiliency**: Added explicit handling for `NotImplementedError` in Anthropic and OpenRouter providers, preventing worker panics when embeddings are misconfigured.
|
|
21
|
+
|
|
22
|
+
## [2.0.0] - 2026-05-29
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
- **.synapignore Support**: Implemented context-aware filtering to prevent traversing symlinks, binary files, and files listed in `.synapignore` or `.gitignore`.
|
|
26
|
+
- **FTS5 Integration**: Shifted semantic full-text search directly to SQLite FTS5 with bounding limits (`LIMIT 50`) for `O(1)` lexical retrieval scaling on 10,000+ file monorepos.
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
- **LLM Hangs in Test Suite**: Isolated testing environment by strictly bypassing real LLM provider configs (`SYNAP_LLM_PROVIDER=""`) preventing test CI suite hangs.
|
|
30
|
+
- **Repository Path Scoping in Hybrid Engine**: Fixed a bug where `repo_path` initialization mistakenly truncated the path to `/tmp` in tests, failing contextual snippet aggregation.
|
|
31
|
+
- **Unbounded BM25 Expansion**: Bounded full-text search token extraction and BM25 results, preventing out-of-memory cascading in CTE traversal during monorepo indexing.
|
|
32
|
+
|
|
8
33
|
## [1.2.3] - 2026-05-28
|
|
9
34
|
|
|
10
35
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: synap-git
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Persistent structural context infrastructure for AI coding agents.
|
|
5
5
|
Project-URL: Homepage, https://github.com/saahilpal/synap-git
|
|
6
6
|
Project-URL: Repository, https://github.com/saahilpal/synap-git
|
|
@@ -52,10 +52,10 @@ Description-Content-Type: text/markdown
|
|
|
52
52
|
|
|
53
53
|
Local Git-aware structural context engine for AI coding agents.
|
|
54
54
|
|
|
55
|
-
[](https://
|
|
57
|
-
[](https://
|
|
58
|
-
[](https://github.com/saahilpal/synapse/actions)
|
|
56
|
+
[](https://pypi.org/project/synap-git/)
|
|
57
|
+
[](https://pypi.org/project/synap-git/)
|
|
58
|
+
[](LICENSE.md)
|
|
59
59
|
|
|
60
60
|
## The Problem
|
|
61
61
|
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
Local Git-aware structural context engine for AI coding agents.
|
|
4
4
|
|
|
5
|
-
[](https://
|
|
7
|
-
[](https://
|
|
8
|
-
[](https://github.com/saahilpal/synapse/actions)
|
|
6
|
+
[](https://pypi.org/project/synap-git/)
|
|
7
|
+
[](https://pypi.org/project/synap-git/)
|
|
8
|
+
[](LICENSE.md)
|
|
9
9
|
|
|
10
10
|
## The Problem
|
|
11
11
|
|
|
@@ -101,8 +101,10 @@ def create_app(runtime: SynapRuntime) -> FastAPI:
|
|
|
101
101
|
target = target[:-3]
|
|
102
102
|
try:
|
|
103
103
|
await asyncio.to_thread(runtime.wiki.ensure_wiki_page, target)
|
|
104
|
-
except Exception:
|
|
105
|
-
|
|
104
|
+
except Exception as e:
|
|
105
|
+
import structlog
|
|
106
|
+
|
|
107
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
106
108
|
|
|
107
109
|
wiki_path = runtime.wiki.wiki_dir / f"{target}.md"
|
|
108
110
|
exists = await asyncio.to_thread(wiki_path.exists)
|
|
@@ -642,8 +642,10 @@ def start(
|
|
|
642
642
|
return
|
|
643
643
|
else:
|
|
644
644
|
pid_file.unlink()
|
|
645
|
-
except Exception:
|
|
646
|
-
|
|
645
|
+
except Exception as e:
|
|
646
|
+
import structlog
|
|
647
|
+
|
|
648
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
647
649
|
|
|
648
650
|
# 2. Spawn detached daemon process
|
|
649
651
|
cmd = [sys.executable, "-m", "synap_git.cli", "daemon-run", abs_path.as_posix()]
|
|
@@ -782,8 +784,10 @@ def stop(
|
|
|
782
784
|
os.waitpid(pid, os.WNOHANG)
|
|
783
785
|
except (ChildProcessError, OSError):
|
|
784
786
|
pass
|
|
785
|
-
except Exception:
|
|
786
|
-
|
|
787
|
+
except Exception as e:
|
|
788
|
+
import structlog
|
|
789
|
+
|
|
790
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
787
791
|
success = not _is_process_running(pid)
|
|
788
792
|
|
|
789
793
|
# Clean up lockfiles
|
|
@@ -895,6 +899,7 @@ def status(
|
|
|
895
899
|
|
|
896
900
|
@app.command()
|
|
897
901
|
def logs(
|
|
902
|
+
path: Annotated[str, typer.Argument(help="Repository path.")] = ".",
|
|
898
903
|
tail: Annotated[
|
|
899
904
|
bool, typer.Option("--tail", "-t", help="Stream new log entries in real-time.")
|
|
900
905
|
] = False,
|
|
@@ -906,7 +911,9 @@ def logs(
|
|
|
906
911
|
"""View and tail Synap runtime logs."""
|
|
907
912
|
import time
|
|
908
913
|
|
|
909
|
-
|
|
914
|
+
settings = _settings(path)
|
|
915
|
+
log_dir = settings.log_path
|
|
916
|
+
assert log_dir is not None
|
|
910
917
|
log_file = log_dir / "daemon.log"
|
|
911
918
|
|
|
912
919
|
if not log_file.exists():
|
|
@@ -938,6 +945,32 @@ def logs(
|
|
|
938
945
|
console.print(f"[red]Error reading logs: {e}[/red]")
|
|
939
946
|
|
|
940
947
|
|
|
948
|
+
@app.command()
|
|
949
|
+
def search(
|
|
950
|
+
query: Annotated[str, typer.Argument(help="Search query.")],
|
|
951
|
+
path: Annotated[str, typer.Option("--path", "-p", help="Repository path.")] = ".",
|
|
952
|
+
max_tokens: Annotated[
|
|
953
|
+
int, typer.Option("--max-tokens", help="Maximum context window tokens.")
|
|
954
|
+
] = 4000,
|
|
955
|
+
) -> None:
|
|
956
|
+
"""Execute a hybrid structural search across the repository."""
|
|
957
|
+
settings = _settings(path)
|
|
958
|
+
runtime = SynapRuntime(settings)
|
|
959
|
+
|
|
960
|
+
with Progress(
|
|
961
|
+
SpinnerColumn(),
|
|
962
|
+
TextColumn("[progress.description]{task.description}"),
|
|
963
|
+
transient=True,
|
|
964
|
+
) as progress:
|
|
965
|
+
progress.add_task(description="Retrieving context...", total=None)
|
|
966
|
+
answer, context, _ = runtime.query_hybrid(query, max_tokens=max_tokens)
|
|
967
|
+
|
|
968
|
+
console.print("\n[bold cyan]Hybrid Retrieval Results[/bold cyan]")
|
|
969
|
+
console.print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
|
970
|
+
console.print(answer)
|
|
971
|
+
console.print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
|
972
|
+
|
|
973
|
+
|
|
941
974
|
@app.command()
|
|
942
975
|
def update() -> None:
|
|
943
976
|
"""Check for updates and upgrade the Synap runtime."""
|
|
@@ -1179,16 +1212,6 @@ def rollback(
|
|
|
1179
1212
|
if not typer.confirm("\nProceed?", default=False):
|
|
1180
1213
|
raise typer.Abort()
|
|
1181
1214
|
|
|
1182
|
-
# Clear current checkpoint
|
|
1183
|
-
try:
|
|
1184
|
-
with runtime.store.connect() as conn:
|
|
1185
|
-
conn.execute(
|
|
1186
|
-
"DELETE FROM checkpoints WHERE branch = ?",
|
|
1187
|
-
(runtime.git.state().effective_branch,),
|
|
1188
|
-
)
|
|
1189
|
-
except Exception:
|
|
1190
|
-
pass
|
|
1191
|
-
|
|
1192
1215
|
# git checkout
|
|
1193
1216
|
try:
|
|
1194
1217
|
subprocess.check_call( # noqa: S603
|
|
@@ -1199,6 +1222,18 @@ def rollback(
|
|
|
1199
1222
|
console.print(f"[red]✗ Git checkout failed: {e}[/red]")
|
|
1200
1223
|
raise typer.Abort()
|
|
1201
1224
|
|
|
1225
|
+
# Clear current checkpoint
|
|
1226
|
+
try:
|
|
1227
|
+
with runtime.store.connect() as conn:
|
|
1228
|
+
conn.execute(
|
|
1229
|
+
"DELETE FROM checkpoints WHERE branch = ?",
|
|
1230
|
+
(runtime.git.state().effective_branch,),
|
|
1231
|
+
)
|
|
1232
|
+
except Exception as e:
|
|
1233
|
+
import structlog
|
|
1234
|
+
|
|
1235
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
1236
|
+
|
|
1202
1237
|
runtime.bootstrap(force=True)
|
|
1203
1238
|
console.print(f"[green]✓ Rolled back successfully to {selected_commit}[/green]")
|
|
1204
1239
|
|
|
@@ -1264,8 +1299,10 @@ def repair(
|
|
|
1264
1299
|
console.print("[3/3] Regenerating wiki from last known state")
|
|
1265
1300
|
try:
|
|
1266
1301
|
runtime.wiki.generate_project_wiki()
|
|
1267
|
-
except Exception:
|
|
1268
|
-
|
|
1302
|
+
except Exception as e:
|
|
1303
|
+
import structlog
|
|
1304
|
+
|
|
1305
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
1269
1306
|
|
|
1270
1307
|
status_info = runtime.status()
|
|
1271
1308
|
console.print(f"\n[green]✓ Recovery complete. {status_info.files} files restored.[/green]")
|
|
@@ -92,7 +92,9 @@ class SynapSettings(BaseSettings):
|
|
|
92
92
|
if line.strip().startswith(f"{key}="):
|
|
93
93
|
return line.split("=", 1)[1].strip().strip("\"'")
|
|
94
94
|
except Exception:
|
|
95
|
-
|
|
95
|
+
import structlog
|
|
96
|
+
|
|
97
|
+
structlog.get_logger(__name__).exception("fallback_credential_lookup_failed", key=key)
|
|
96
98
|
return None
|
|
97
99
|
|
|
98
100
|
@property
|
|
@@ -22,7 +22,6 @@ class TraceResult:
|
|
|
22
22
|
|
|
23
23
|
def configure_logging(settings: SynapSettings) -> None:
|
|
24
24
|
from logging.handlers import RotatingFileHandler
|
|
25
|
-
from pathlib import Path
|
|
26
25
|
|
|
27
26
|
timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True)
|
|
28
27
|
shared_processors: list[structlog.typing.Processor] = [
|
|
@@ -38,7 +37,8 @@ def configure_logging(settings: SynapSettings) -> None:
|
|
|
38
37
|
else:
|
|
39
38
|
renderer = structlog.dev.ConsoleRenderer(colors=False)
|
|
40
39
|
|
|
41
|
-
log_dir =
|
|
40
|
+
log_dir = settings.log_path
|
|
41
|
+
assert log_dir is not None
|
|
42
42
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
43
43
|
log_file = log_dir / "daemon.log"
|
|
44
44
|
|
|
@@ -128,8 +128,10 @@ class GitRepository:
|
|
|
128
128
|
break
|
|
129
129
|
if is_revert:
|
|
130
130
|
return GitChange(kind=GitChangeKind.REVERT, previous=previous, current=current)
|
|
131
|
-
except Exception:
|
|
132
|
-
|
|
131
|
+
except Exception as e:
|
|
132
|
+
import structlog
|
|
133
|
+
|
|
134
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
133
135
|
|
|
134
136
|
if len(current.commit_parent_hashes) > 1:
|
|
135
137
|
return GitChange(kind=GitChangeKind.MERGE, previous=previous, current=current)
|
|
@@ -244,16 +244,20 @@ class RuntimeDaemon:
|
|
|
244
244
|
if heartbeat_file.exists():
|
|
245
245
|
try:
|
|
246
246
|
heartbeat_file.unlink()
|
|
247
|
-
except Exception:
|
|
248
|
-
|
|
247
|
+
except Exception as e:
|
|
248
|
+
import structlog
|
|
249
|
+
|
|
250
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
249
251
|
|
|
250
252
|
# Cleanup process PID lockfile as well
|
|
251
253
|
pid_file = self.settings.repository_path / ".synap" / "daemon.pid"
|
|
252
254
|
if pid_file.exists():
|
|
253
255
|
try:
|
|
254
256
|
pid_file.unlink()
|
|
255
|
-
except Exception:
|
|
256
|
-
|
|
257
|
+
except Exception as e:
|
|
258
|
+
import structlog
|
|
259
|
+
|
|
260
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
257
261
|
|
|
258
262
|
async def _poll_git_loop(self) -> None:
|
|
259
263
|
while not self._stop_event.is_set():
|
|
@@ -69,6 +69,7 @@ class SynapRuntime:
|
|
|
69
69
|
self.parser_registry = CodeParserRegistry()
|
|
70
70
|
self.llm_provider = get_llm_provider(settings)
|
|
71
71
|
self.retrieval_engine = HybridRetrievalEngine(
|
|
72
|
+
repo_path=self.settings.repository_path,
|
|
72
73
|
store=self.store,
|
|
73
74
|
llm_provider=self.llm_provider,
|
|
74
75
|
trace_store=self.trace_store,
|
|
@@ -140,8 +141,10 @@ class SynapRuntime:
|
|
|
140
141
|
if change.kind == "revert":
|
|
141
142
|
self.handle_revert(prev, git_state)
|
|
142
143
|
return git_state.head_commit
|
|
143
|
-
except Exception:
|
|
144
|
-
|
|
144
|
+
except Exception as e:
|
|
145
|
+
import structlog
|
|
146
|
+
|
|
147
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
145
148
|
|
|
146
149
|
return self.index_repository(git_state=git_state, force=force)
|
|
147
150
|
|
|
@@ -244,6 +247,53 @@ class SynapRuntime:
|
|
|
244
247
|
|
|
245
248
|
return commit
|
|
246
249
|
|
|
250
|
+
def _schedule_embedding(
|
|
251
|
+
self, file_id: str, symbols: list[dict[str, Any]], content_hash: str
|
|
252
|
+
) -> None:
|
|
253
|
+
provider = self.llm_provider
|
|
254
|
+
if not provider:
|
|
255
|
+
return
|
|
256
|
+
|
|
257
|
+
def _worker() -> None:
|
|
258
|
+
import structlog
|
|
259
|
+
|
|
260
|
+
logger = structlog.get_logger()
|
|
261
|
+
try:
|
|
262
|
+
batches = [symbols[i : i + 50] for i in range(0, len(symbols), 50)]
|
|
263
|
+
model_name = getattr(provider, "default_model", "unknown")
|
|
264
|
+
for batch in batches:
|
|
265
|
+
for sym in batch:
|
|
266
|
+
try:
|
|
267
|
+
# Avoid re-embedding if unchanged?
|
|
268
|
+
text = (
|
|
269
|
+
sym["name"] + " " + (sym.get("metadata") or {}).get("docstring", "")
|
|
270
|
+
)
|
|
271
|
+
vector = provider.embed(text)
|
|
272
|
+
self.store.put_embedding(
|
|
273
|
+
sym["symbol_id"], model_name, "1.0", "1.0", vector, content_hash
|
|
274
|
+
)
|
|
275
|
+
except NotImplementedError as e:
|
|
276
|
+
logger.warning(
|
|
277
|
+
"embedding_not_supported_by_provider",
|
|
278
|
+
error=str(e),
|
|
279
|
+
suggestion="Configure SYNAP_EMBED_PROVIDER=ollama or openai.",
|
|
280
|
+
)
|
|
281
|
+
return # Stop attempting batch if not supported
|
|
282
|
+
except Exception as e:
|
|
283
|
+
logger.warning("background_embed_failed_for_symbol", error=str(e))
|
|
284
|
+
except Exception as e:
|
|
285
|
+
logger.warning("background_embed_failed", error=str(e))
|
|
286
|
+
|
|
287
|
+
try:
|
|
288
|
+
import asyncio
|
|
289
|
+
|
|
290
|
+
loop = asyncio.get_running_loop()
|
|
291
|
+
loop.create_task(asyncio.to_thread(_worker))
|
|
292
|
+
except RuntimeError:
|
|
293
|
+
import threading
|
|
294
|
+
|
|
295
|
+
threading.Thread(target=_worker, daemon=True).start()
|
|
296
|
+
|
|
247
297
|
def _first_run_index(self, git_state: GitState) -> str | None:
|
|
248
298
|
self.logger.info("first_run_indexing_started", branch=git_state.effective_branch)
|
|
249
299
|
import time
|
|
@@ -254,16 +304,9 @@ class SynapRuntime:
|
|
|
254
304
|
repository_path=self.settings.repository_path,
|
|
255
305
|
max_file_bytes=self.settings.max_file_bytes,
|
|
256
306
|
)
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
files_to_parse = []
|
|
260
|
-
for file_info in scan.files:
|
|
261
|
-
files_to_parse.append((file_info.path, file_info.relative_path, file_info.content))
|
|
262
|
-
|
|
263
|
-
num_files = len(files_to_parse)
|
|
307
|
+
num_files = scanner.count_files()
|
|
264
308
|
chunk_size = 500
|
|
265
309
|
parsed_results = []
|
|
266
|
-
|
|
267
310
|
# Parallel parsing using ProcessPoolExecutor
|
|
268
311
|
import multiprocessing
|
|
269
312
|
from concurrent.futures import ProcessPoolExecutor
|
|
@@ -301,16 +344,22 @@ class SynapRuntime:
|
|
|
301
344
|
task = progress.add_task("[yellow]Parsing codebase...[/yellow]", total=num_files)
|
|
302
345
|
|
|
303
346
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
304
|
-
|
|
305
|
-
|
|
347
|
+
from itertools import islice
|
|
348
|
+
|
|
349
|
+
file_iterator = scanner.scan()
|
|
350
|
+
while True:
|
|
351
|
+
chunk_files = list(islice(file_iterator, chunk_size))
|
|
352
|
+
if not chunk_files:
|
|
353
|
+
break
|
|
354
|
+
|
|
355
|
+
file_info_map = {f.relative_path: f for f in chunk_files}
|
|
356
|
+
chunk = [(f.path, f.relative_path, f.content) for f in chunk_files]
|
|
306
357
|
futures = [executor.submit(_parse_worker, item) for item in chunk]
|
|
307
358
|
chunk_results = [f.result() for f in futures]
|
|
308
359
|
|
|
309
360
|
for res in chunk_results:
|
|
310
361
|
rel_path = res["path"]
|
|
311
|
-
matching_info =
|
|
312
|
-
(f for f in scan.files if f.relative_path == rel_path), None
|
|
313
|
-
)
|
|
362
|
+
matching_info = file_info_map.get(rel_path)
|
|
314
363
|
git_oid = matching_info.git_oid if matching_info else ""
|
|
315
364
|
content_hash = matching_info.content_hash if matching_info else ""
|
|
316
365
|
|
|
@@ -328,6 +377,7 @@ class SynapRuntime:
|
|
|
328
377
|
language=res["language"],
|
|
329
378
|
symbols=res["symbols"],
|
|
330
379
|
)
|
|
380
|
+
self._schedule_embedding(file_id_hash, res["symbols"], content_hash)
|
|
331
381
|
self.store.enqueue_wiki(rel_path)
|
|
332
382
|
parsed_results.append((file_id_hash, rel_path, res["imports"]))
|
|
333
383
|
|
|
@@ -336,20 +386,26 @@ class SynapRuntime:
|
|
|
336
386
|
self._resolve_and_insert_edges(parsed_results)
|
|
337
387
|
parsed_results.clear()
|
|
338
388
|
|
|
339
|
-
progress.advance(task, len(
|
|
389
|
+
progress.advance(task, len(chunk_files))
|
|
340
390
|
del chunk_results
|
|
341
391
|
else:
|
|
342
392
|
with ProcessPoolExecutor(max_workers=num_workers) as executor:
|
|
343
|
-
|
|
344
|
-
|
|
393
|
+
from itertools import islice
|
|
394
|
+
|
|
395
|
+
file_iterator = scanner.scan()
|
|
396
|
+
while True:
|
|
397
|
+
chunk_files = list(islice(file_iterator, chunk_size))
|
|
398
|
+
if not chunk_files:
|
|
399
|
+
break
|
|
400
|
+
|
|
401
|
+
file_info_map = {f.relative_path: f for f in chunk_files}
|
|
402
|
+
chunk = [(f.path, f.relative_path, f.content) for f in chunk_files]
|
|
345
403
|
futures = [executor.submit(_parse_worker, item) for item in chunk]
|
|
346
404
|
chunk_results = [f.result() for f in futures]
|
|
347
405
|
|
|
348
406
|
for res in chunk_results:
|
|
349
407
|
rel_path = res["path"]
|
|
350
|
-
matching_info =
|
|
351
|
-
(f for f in scan.files if f.relative_path == rel_path), None
|
|
352
|
-
)
|
|
408
|
+
matching_info = file_info_map.get(rel_path)
|
|
353
409
|
git_oid = matching_info.git_oid if matching_info else ""
|
|
354
410
|
content_hash = matching_info.content_hash if matching_info else ""
|
|
355
411
|
|
|
@@ -367,6 +423,7 @@ class SynapRuntime:
|
|
|
367
423
|
language=res["language"],
|
|
368
424
|
symbols=res["symbols"],
|
|
369
425
|
)
|
|
426
|
+
self._schedule_embedding(file_id_hash, res["symbols"], content_hash)
|
|
370
427
|
self.store.enqueue_wiki(rel_path)
|
|
371
428
|
parsed_results.append((file_id_hash, rel_path, res["imports"]))
|
|
372
429
|
|
|
@@ -426,6 +483,7 @@ class SynapRuntime:
|
|
|
426
483
|
|
|
427
484
|
added_or_modified = []
|
|
428
485
|
deleted = []
|
|
486
|
+
renames = []
|
|
429
487
|
for line in res.stdout.splitlines():
|
|
430
488
|
if not line.strip():
|
|
431
489
|
continue
|
|
@@ -433,13 +491,41 @@ class SynapRuntime:
|
|
|
433
491
|
if len(parts) >= 2:
|
|
434
492
|
status = parts[0]
|
|
435
493
|
if status.startswith("R"):
|
|
436
|
-
|
|
494
|
+
renames.append((parts[1], parts[2]))
|
|
437
495
|
added_or_modified.append(parts[2])
|
|
438
496
|
elif status == "D":
|
|
439
497
|
deleted.append(parts[1])
|
|
440
498
|
else:
|
|
441
499
|
added_or_modified.append(parts[1])
|
|
442
500
|
|
|
501
|
+
import hashlib
|
|
502
|
+
|
|
503
|
+
# Apply renames to preserve identity
|
|
504
|
+
with self.store.connect() as conn:
|
|
505
|
+
for old_path, new_path in renames:
|
|
506
|
+
file_row = conn.execute(
|
|
507
|
+
"SELECT file_id, content_hash FROM files WHERE path = ?", (old_path,)
|
|
508
|
+
).fetchone()
|
|
509
|
+
if file_row:
|
|
510
|
+
old_file_id = file_row["file_id"]
|
|
511
|
+
content_hash = file_row["content_hash"]
|
|
512
|
+
new_file_id = hashlib.sha256(
|
|
513
|
+
(new_path + content_hash).encode("utf-8")
|
|
514
|
+
).hexdigest()
|
|
515
|
+
|
|
516
|
+
conn.execute("PRAGMA foreign_keys=OFF")
|
|
517
|
+
try:
|
|
518
|
+
conn.execute(
|
|
519
|
+
"UPDATE files SET file_id = ?, path = ? WHERE file_id = ?",
|
|
520
|
+
(new_file_id, new_path, old_file_id),
|
|
521
|
+
)
|
|
522
|
+
conn.execute(
|
|
523
|
+
"UPDATE symbols SET file_id = ? WHERE file_id = ?",
|
|
524
|
+
(new_file_id, old_file_id),
|
|
525
|
+
)
|
|
526
|
+
finally:
|
|
527
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
528
|
+
|
|
443
529
|
# Filter added/modified files
|
|
444
530
|
scanner = RepositoryScanner(
|
|
445
531
|
repository_path=self.settings.repository_path,
|
|
@@ -549,6 +635,7 @@ class SynapRuntime:
|
|
|
549
635
|
language=parse_result.language,
|
|
550
636
|
symbols=symbols_list,
|
|
551
637
|
)
|
|
638
|
+
self._schedule_embedding(file_id_hash, symbols_list, content_hash)
|
|
552
639
|
|
|
553
640
|
self.store.set_wiki_status(rel_path, None, "stale")
|
|
554
641
|
self.store.enqueue_wiki(rel_path)
|
|
@@ -600,8 +687,12 @@ class SynapRuntime:
|
|
|
600
687
|
candidate_rel = (target_dir / "/".join(parts)).as_posix()
|
|
601
688
|
for ext in [".py", ".ts", ".go", ".rs"]:
|
|
602
689
|
candidate_paths.add(f"{candidate_rel}{ext}")
|
|
603
|
-
except Exception:
|
|
604
|
-
|
|
690
|
+
except Exception as e:
|
|
691
|
+
import structlog
|
|
692
|
+
|
|
693
|
+
structlog.get_logger().error(
|
|
694
|
+
"suppressed_error_caught", error=str(e), exc_info=True
|
|
695
|
+
)
|
|
605
696
|
else:
|
|
606
697
|
module_keys.add(module_part)
|
|
607
698
|
fts_names.add(target_name)
|
|
@@ -667,8 +758,12 @@ class SynapRuntime:
|
|
|
667
758
|
if cand_path in path_to_file_id:
|
|
668
759
|
target_file_id = path_to_file_id[cand_path]
|
|
669
760
|
break
|
|
670
|
-
except Exception:
|
|
671
|
-
|
|
761
|
+
except Exception as e:
|
|
762
|
+
import structlog
|
|
763
|
+
|
|
764
|
+
structlog.get_logger().error(
|
|
765
|
+
"suppressed_error_caught", error=str(e), exc_info=True
|
|
766
|
+
)
|
|
672
767
|
else:
|
|
673
768
|
target_file_id = module_key_to_file_id.get(module_part)
|
|
674
769
|
|
|
@@ -701,11 +796,23 @@ class SynapRuntime:
|
|
|
701
796
|
for row in rows:
|
|
702
797
|
fts_symbols_by_name.setdefault(row["name"], []).append(dict(row))
|
|
703
798
|
|
|
799
|
+
target_symbols_dict: dict[str, dict[str, list[dict[str, Any]]]] = {}
|
|
800
|
+
for t_file_id, syms in target_symbols_by_file.items():
|
|
801
|
+
name_map: dict[str, Any] = {}
|
|
802
|
+
for sym in syms:
|
|
803
|
+
name_map.setdefault(sym["name"], []).append(sym)
|
|
804
|
+
target_symbols_dict[t_file_id] = name_map
|
|
805
|
+
|
|
704
806
|
for file_id, rel_path, imports in parsed_results:
|
|
705
807
|
file_symbols = file_symbols_map.get(file_id, [])
|
|
706
808
|
current_path = Path(rel_path)
|
|
809
|
+
edges_added_for_file = 0
|
|
810
|
+
max_edges_per_file = 1000
|
|
707
811
|
|
|
708
812
|
for imp in imports:
|
|
813
|
+
if edges_added_for_file >= max_edges_per_file:
|
|
814
|
+
break
|
|
815
|
+
|
|
709
816
|
target_file_id = None
|
|
710
817
|
if ":" in imp:
|
|
711
818
|
module_part, target_name = imp.split(":", 1)
|
|
@@ -723,20 +830,36 @@ class SynapRuntime:
|
|
|
723
830
|
if cand_path in path_to_file_id:
|
|
724
831
|
target_file_id = path_to_file_id[cand_path]
|
|
725
832
|
break
|
|
726
|
-
except Exception:
|
|
727
|
-
|
|
833
|
+
except Exception as e:
|
|
834
|
+
import structlog
|
|
835
|
+
|
|
836
|
+
structlog.get_logger().error(
|
|
837
|
+
"suppressed_error_caught", error=str(e), exc_info=True
|
|
838
|
+
)
|
|
728
839
|
else:
|
|
729
840
|
target_file_id = module_key_to_file_id.get(module_part)
|
|
730
841
|
|
|
842
|
+
# Fuzzy fallback
|
|
843
|
+
if not target_file_id:
|
|
844
|
+
for mk, fid in module_key_to_file_id.items():
|
|
845
|
+
if mk.endswith(module_part) or mk.endswith(f".{module_part}"):
|
|
846
|
+
target_file_id = fid
|
|
847
|
+
break
|
|
848
|
+
|
|
731
849
|
target_symbols = []
|
|
732
850
|
if target_file_id:
|
|
733
|
-
|
|
734
|
-
|
|
851
|
+
target_symbols = target_symbols_dict.get(target_file_id, {}).get(
|
|
852
|
+
target_name, []
|
|
853
|
+
)
|
|
735
854
|
else:
|
|
736
855
|
target_symbols = fts_symbols_by_name.get(target_name, [])
|
|
737
856
|
|
|
738
857
|
for ts in target_symbols:
|
|
858
|
+
if edges_added_for_file >= max_edges_per_file:
|
|
859
|
+
break
|
|
739
860
|
for fs in file_symbols:
|
|
861
|
+
if edges_added_for_file >= max_edges_per_file:
|
|
862
|
+
break
|
|
740
863
|
if fs["symbol_id"] == ts["symbol_id"]:
|
|
741
864
|
continue
|
|
742
865
|
edge_id = stable_hash(
|
|
@@ -747,6 +870,7 @@ class SynapRuntime:
|
|
|
747
870
|
}
|
|
748
871
|
)
|
|
749
872
|
edges_to_insert.append((edge_id, fs["symbol_id"], ts["symbol_id"]))
|
|
873
|
+
edges_added_for_file += 1
|
|
750
874
|
|
|
751
875
|
if edges_to_insert:
|
|
752
876
|
conn.executemany(
|
|
@@ -787,8 +911,10 @@ class SynapRuntime:
|
|
|
787
911
|
is_dirty = False
|
|
788
912
|
try:
|
|
789
913
|
is_dirty = self.git.state().is_dirty
|
|
790
|
-
except Exception:
|
|
791
|
-
|
|
914
|
+
except Exception as e:
|
|
915
|
+
import structlog
|
|
916
|
+
|
|
917
|
+
structlog.get_logger().error("suppressed_error_caught", error=str(e), exc_info=True)
|
|
792
918
|
return self.retrieval_engine.retrieve(query, max_tokens=max_tokens, is_dirty=is_dirty)
|
|
793
919
|
|
|
794
920
|
def doctor(self, *, auto_recover: bool = False) -> dict[str, Any]:
|