codecortex 0.13.1__tar.gz → 0.14.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codecortex-0.13.1/src/codecortex.egg-info → codecortex-0.14.2}/PKG-INFO +24 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/README.md +23 -0
- {codecortex-0.13.1 → codecortex-0.14.2/src/codecortex.egg-info}/PKG-INFO +24 -1
- codecortex-0.14.2/src/codeintel/__init__.py +1 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/auth.py +35 -4
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/cache.py +12 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/_common.py +11 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/graph.py +10 -3
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/index.py +14 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/map.py +8 -2
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/setup.py +7 -2
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/status.py +5 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/gateway.py +57 -13
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/http_server.py +7 -3
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/indexer.py +80 -2
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/injector.py +36 -7
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/installer.py +63 -4
- codecortex-0.14.2/src/codeintel/policy.py +93 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/provider.py +4 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/providers/graph.py +23 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/reindexer.py +53 -5
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/reset.py +48 -3
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/semantic_db.py +5 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/server.py +14 -0
- codecortex-0.14.2/tests/test_cache.py +109 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_chunking.py +36 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_cli_commands.py +42 -3
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_gateway.py +193 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_graph_provider.py +22 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_graph_stdin.py +29 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_installer.py +82 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_mapper.py +56 -0
- codecortex-0.14.2/tests/test_rbac.py +438 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_reindexer.py +83 -1
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_reset.py +44 -0
- codecortex-0.13.1/src/codeintel/__init__.py +0 -1
- codecortex-0.13.1/src/codeintel/policy.py +0 -28
- codecortex-0.13.1/tests/test_cache.py +0 -61
- codecortex-0.13.1/tests/test_rbac.py +0 -178
- {codecortex-0.13.1 → codecortex-0.14.2}/LICENSE +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/pyproject.toml +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/setup.cfg +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codecortex.egg-info/SOURCES.txt +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codecortex.egg-info/dependency_links.txt +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codecortex.egg-info/entry_points.txt +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codecortex.egg-info/requires.txt +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codecortex.egg-info/top_level.txt +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/__main__.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/__init__.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/doctor.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/gen_token.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/install.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/query.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/reset.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/serve.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/commands/serve_http.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/config.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/doctor.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/grapher.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/logconfig.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/mapper.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/metrics.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/onboarding.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/providers/__init__.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/providers/lsp.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/providers/none.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/providers/semantic.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/py.typed +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/searcher.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/term.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/verify.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/viewer/__init__.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/src/codeintel/viewer/graph_template.html +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_cli_help.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_config.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_doctor.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_e2e.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_engine_adoption.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_enterprise.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_graph_real.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_grapher.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_hardening.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_http_auth.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_http_server.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_integration.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_lsp_provider.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_lsp_real.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_mcp_handshake.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_mcp_server.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_model_dimension.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_never_raise.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_onboarding.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_rerank.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_semantic_provider.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_term.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_treesitter.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_verify.py +0 -0
- {codecortex-0.13.1 → codecortex-0.14.2}/tests/test_verify_call.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: codecortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.2
|
|
4
4
|
Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
|
|
5
5
|
Author: Shammai Hamilton
|
|
6
6
|
License-Expression: MIT
|
|
@@ -296,6 +296,12 @@ codeintel index /path/to/repo
|
|
|
296
296
|
The reply names the fix when it can: a missing symbol now returns a `hint` with the exact command
|
|
297
297
|
rather than a bare reason.
|
|
298
298
|
|
|
299
|
+
One more honest caveat. For targets that are **symbol names or free text** (most of them —
|
|
300
|
+
`callers`, `impact`, `hotspots`, `search`), there is no file whose content hash could change, so a
|
|
301
|
+
cached answer is invalidated only when a background reindex completes, and those are debounced
|
|
302
|
+
(~30s). An edit followed immediately by the same query can therefore return the pre-edit answer.
|
|
303
|
+
Targets that are real file paths are content-hashed and refresh as soon as the bytes change.
|
|
304
|
+
|
|
299
305
|
Pass `--engine auto` (the default) and codeintel chooses the best engine per operation. Pass `--engine both` or `--engine all` to fan out to multiple engines and merge results.
|
|
300
306
|
|
|
301
307
|
## Documentation
|
|
@@ -433,6 +439,23 @@ docker build -t codeintel . && docker run -p 127.0.0.1:8766:8766 \
|
|
|
433
439
|
-e CODEINTEL_HTTP_TOKEN="$(openssl rand -hex 32)" codeintel
|
|
434
440
|
```
|
|
435
441
|
|
|
442
|
+
## Reporting a problem
|
|
443
|
+
|
|
444
|
+
`codeintel doctor --json` prints a complete, machine-readable picture of what's installed, what's
|
|
445
|
+
runnable, and whether this repo is indexed — per engine, with the remediation for each gap. Paste
|
|
446
|
+
it into an issue and the report is actionable immediately instead of needing a round trip:
|
|
447
|
+
|
|
448
|
+
```bash
|
|
449
|
+
codeintel doctor --json
|
|
450
|
+
```
|
|
451
|
+
|
|
452
|
+
It reports only local engine and index state. Over the HTTP transport the `registrations` field —
|
|
453
|
+
which names agent config files on the machine running the server — is deliberately omitted.
|
|
454
|
+
|
|
455
|
+
If a *result* looks wrong rather than a command failing, include the exact `code.query` call and
|
|
456
|
+
its full envelope. `reason`, `hint`, `engine`, `cached`, and `reindexing` between them explain
|
|
457
|
+
which engine answered and how current its index was, which is usually the whole diagnosis.
|
|
458
|
+
|
|
436
459
|
## Development
|
|
437
460
|
|
|
438
461
|
```bash
|
|
@@ -259,6 +259,12 @@ codeintel index /path/to/repo
|
|
|
259
259
|
The reply names the fix when it can: a missing symbol now returns a `hint` with the exact command
|
|
260
260
|
rather than a bare reason.
|
|
261
261
|
|
|
262
|
+
One more honest caveat. For targets that are **symbol names or free text** (most of them —
|
|
263
|
+
`callers`, `impact`, `hotspots`, `search`), there is no file whose content hash could change, so a
|
|
264
|
+
cached answer is invalidated only when a background reindex completes, and those are debounced
|
|
265
|
+
(~30s). An edit followed immediately by the same query can therefore return the pre-edit answer.
|
|
266
|
+
Targets that are real file paths are content-hashed and refresh as soon as the bytes change.
|
|
267
|
+
|
|
262
268
|
Pass `--engine auto` (the default) and codeintel chooses the best engine per operation. Pass `--engine both` or `--engine all` to fan out to multiple engines and merge results.
|
|
263
269
|
|
|
264
270
|
## Documentation
|
|
@@ -396,6 +402,23 @@ docker build -t codeintel . && docker run -p 127.0.0.1:8766:8766 \
|
|
|
396
402
|
-e CODEINTEL_HTTP_TOKEN="$(openssl rand -hex 32)" codeintel
|
|
397
403
|
```
|
|
398
404
|
|
|
405
|
+
## Reporting a problem
|
|
406
|
+
|
|
407
|
+
`codeintel doctor --json` prints a complete, machine-readable picture of what's installed, what's
|
|
408
|
+
runnable, and whether this repo is indexed — per engine, with the remediation for each gap. Paste
|
|
409
|
+
it into an issue and the report is actionable immediately instead of needing a round trip:
|
|
410
|
+
|
|
411
|
+
```bash
|
|
412
|
+
codeintel doctor --json
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
It reports only local engine and index state. Over the HTTP transport the `registrations` field —
|
|
416
|
+
which names agent config files on the machine running the server — is deliberately omitted.
|
|
417
|
+
|
|
418
|
+
If a *result* looks wrong rather than a command failing, include the exact `code.query` call and
|
|
419
|
+
its full envelope. `reason`, `hint`, `engine`, `cached`, and `reindexing` between them explain
|
|
420
|
+
which engine answered and how current its index was, which is usually the whole diagnosis.
|
|
421
|
+
|
|
399
422
|
## Development
|
|
400
423
|
|
|
401
424
|
```bash
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: codecortex
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.14.2
|
|
4
4
|
Summary: Local-first, MCP-native code-intelligence server — graph, LSP, and semantic search behind one safe code.query tool for coding agents.
|
|
5
5
|
Author: Shammai Hamilton
|
|
6
6
|
License-Expression: MIT
|
|
@@ -296,6 +296,12 @@ codeintel index /path/to/repo
|
|
|
296
296
|
The reply names the fix when it can: a missing symbol now returns a `hint` with the exact command
|
|
297
297
|
rather than a bare reason.
|
|
298
298
|
|
|
299
|
+
One more honest caveat. For targets that are **symbol names or free text** (most of them —
|
|
300
|
+
`callers`, `impact`, `hotspots`, `search`), there is no file whose content hash could change, so a
|
|
301
|
+
cached answer is invalidated only when a background reindex completes, and those are debounced
|
|
302
|
+
(~30s). An edit followed immediately by the same query can therefore return the pre-edit answer.
|
|
303
|
+
Targets that are real file paths are content-hashed and refresh as soon as the bytes change.
|
|
304
|
+
|
|
299
305
|
Pass `--engine auto` (the default) and codeintel chooses the best engine per operation. Pass `--engine both` or `--engine all` to fan out to multiple engines and merge results.
|
|
300
306
|
|
|
301
307
|
## Documentation
|
|
@@ -433,6 +439,23 @@ docker build -t codeintel . && docker run -p 127.0.0.1:8766:8766 \
|
|
|
433
439
|
-e CODEINTEL_HTTP_TOKEN="$(openssl rand -hex 32)" codeintel
|
|
434
440
|
```
|
|
435
441
|
|
|
442
|
+
## Reporting a problem
|
|
443
|
+
|
|
444
|
+
`codeintel doctor --json` prints a complete, machine-readable picture of what's installed, what's
|
|
445
|
+
runnable, and whether this repo is indexed — per engine, with the remediation for each gap. Paste
|
|
446
|
+
it into an issue and the report is actionable immediately instead of needing a round trip:
|
|
447
|
+
|
|
448
|
+
```bash
|
|
449
|
+
codeintel doctor --json
|
|
450
|
+
```
|
|
451
|
+
|
|
452
|
+
It reports only local engine and index state. Over the HTTP transport the `registrations` field —
|
|
453
|
+
which names agent config files on the machine running the server — is deliberately omitted.
|
|
454
|
+
|
|
455
|
+
If a *result* looks wrong rather than a command failing, include the exact `code.query` call and
|
|
456
|
+
its full envelope. `reason`, `hint`, `engine`, `cached`, and `reindexing` between them explain
|
|
457
|
+
which engine answered and how current its index was, which is usually the whole diagnosis.
|
|
458
|
+
|
|
436
459
|
## Development
|
|
437
460
|
|
|
438
461
|
```bash
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.14.2"
|
|
@@ -38,9 +38,11 @@ class TokenAuth:
|
|
|
38
38
|
"""Resolves a bearer token to a role and builds the matching op policy. Tokens are held as
|
|
39
39
|
sha256 hashes; a presented token is hashed and looked up in O(1). Immutable; thread-safe."""
|
|
40
40
|
|
|
41
|
-
def __init__(self, token_hash_to_role: dict[str, str], role_ops: dict[str, list[str]]
|
|
41
|
+
def __init__(self, token_hash_to_role: dict[str, str], role_ops: dict[str, list[str]],
|
|
42
|
+
role_roots: dict[str, list[str]] | None = None) -> None:
|
|
42
43
|
self._tokens = token_hash_to_role
|
|
43
44
|
self._role_ops = role_ops
|
|
45
|
+
self._role_roots = role_roots or {}
|
|
44
46
|
|
|
45
47
|
@property
|
|
46
48
|
def enabled(self) -> bool:
|
|
@@ -56,13 +58,19 @@ class TokenAuth:
|
|
|
56
58
|
def build_policy(self) -> TieringPolicy:
|
|
57
59
|
"""A role with ops ``["*"]`` (or none) is unrestricted → omit it from the rules, since
|
|
58
60
|
TieringPolicy treats a role absent from its rules as full-access. Every other role maps to
|
|
59
|
-
its explicit op allowlist (an empty list = deny all ops, a fail-safe default).
|
|
61
|
+
its explicit op allowlist (an empty list = deny all ops, a fail-safe default).
|
|
62
|
+
|
|
63
|
+
``enabled`` is True whenever RBAC is configured at all — NOT merely when some role has a
|
|
64
|
+
restricted op list. It used to be ``bool(rules)``, which meant a config whose roles were
|
|
65
|
+
all ``["*"]`` produced a disabled policy, and a disabled policy enforces no root scoping
|
|
66
|
+
either. Op behavior is unchanged (a role absent from rules is still unrestricted); this
|
|
67
|
+
only ensures the root allowlist is actually consulted."""
|
|
60
68
|
rules: dict[str, list[str]] = {}
|
|
61
69
|
for role, ops in self._role_ops.items():
|
|
62
70
|
if _ALL in ops:
|
|
63
71
|
continue # unrestricted
|
|
64
72
|
rules[role] = list(ops)
|
|
65
|
-
return TieringPolicy(enabled=
|
|
73
|
+
return TieringPolicy(enabled=True, rules=rules, roots=dict(self._role_roots))
|
|
66
74
|
|
|
67
75
|
|
|
68
76
|
def _auth_config_path() -> pathlib.Path | None:
|
|
@@ -99,6 +107,19 @@ def load_auth() -> TokenAuth:
|
|
|
99
107
|
logger.warning("auth: role %r ops must be a list (got %r) — denying all ops for it", role, ops)
|
|
100
108
|
role_ops[str(role)] = []
|
|
101
109
|
|
|
110
|
+
roots_section = data.get("roots")
|
|
111
|
+
raw_roots: dict = roots_section if isinstance(roots_section, dict) else {}
|
|
112
|
+
role_roots: dict[str, list[str]] = {}
|
|
113
|
+
for role, paths in raw_roots.items():
|
|
114
|
+
if isinstance(paths, list):
|
|
115
|
+
role_roots[str(role)] = [str(r) for r in paths]
|
|
116
|
+
else:
|
|
117
|
+
# Same fail-CLOSED rule as ops: a bare string (`reader = "/srv/repo"`) is a typo, and
|
|
118
|
+
# guessing it meant a one-element list would hand out access the operator never wrote.
|
|
119
|
+
logger.warning("auth: role %r roots must be a list (got %r) — denying all paths for it",
|
|
120
|
+
role, paths)
|
|
121
|
+
role_roots[str(role)] = []
|
|
122
|
+
|
|
102
123
|
tokens_section = data.get("tokens")
|
|
103
124
|
raw_tokens: dict = tokens_section if isinstance(tokens_section, dict) else {}
|
|
104
125
|
token_hash_to_role: dict[str, str] = {}
|
|
@@ -107,6 +128,7 @@ def load_auth() -> TokenAuth:
|
|
|
107
128
|
# A token mapped to an undefined role fails safe: define it as deny-all (empty op list).
|
|
108
129
|
if role not in role_ops:
|
|
109
130
|
role_ops[role] = []
|
|
131
|
+
role_roots.setdefault(role, [])
|
|
110
132
|
tok = str(tok)
|
|
111
133
|
if tok[:len(_SHA_PREFIX)].lower() == _SHA_PREFIX: # case-insensitive `sha256:` prefix
|
|
112
134
|
h = tok[len(_SHA_PREFIX):].strip().lower()
|
|
@@ -118,4 +140,13 @@ def load_auth() -> TokenAuth:
|
|
|
118
140
|
if not token_hash_to_role:
|
|
119
141
|
logger.warning("auth: %s defines no usable [tokens] — RBAC is OFF (no token→role mapping)", path)
|
|
120
142
|
|
|
121
|
-
|
|
143
|
+
# A configured-but-unscoped RBAC deployment is the exact shape of the hole this closes, so say
|
|
144
|
+
# so at load time rather than letting the operator discover it as a wall of 403s.
|
|
145
|
+
unscoped = sorted(r for r in role_ops if not role_roots.get(r))
|
|
146
|
+
if token_hash_to_role and unscoped:
|
|
147
|
+
logger.warning(
|
|
148
|
+
"auth: roles %s have no [roots] entry and may target NO project — add a [roots] table "
|
|
149
|
+
"(e.g. `%s = [\"/srv/repos/team-a\"]`, or `= [\"*\"]` for unrestricted)",
|
|
150
|
+
", ".join(unscoped), unscoped[0])
|
|
151
|
+
|
|
152
|
+
return TokenAuth(token_hash_to_role, role_ops, role_roots)
|
|
@@ -9,9 +9,20 @@ from codeintel.provider import Result
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def _compute_hash(target: str, project_root: str) -> str:
|
|
12
|
+
"""The cache key's content component: a file target hashes its BYTES, so an edit invalidates.
|
|
13
|
+
|
|
14
|
+
A relative target must be resolved against ``project_root``, not the process cwd. It used to
|
|
15
|
+
use ``os.path.realpath(target)`` alone — and the ops whose target is a path take it
|
|
16
|
+
repo-relative (`overview` passes it straight through as serena's ``relative_path``). The
|
|
17
|
+
server is a long-lived singleton answering for whichever ``project_root`` a caller names, so
|
|
18
|
+
its cwd matches at most one repo and usually none: `inside_root` was false, and every such
|
|
19
|
+
entry silently fell back to hashing the *string*, which never changes when the file does.
|
|
20
|
+
That is exactly the "an edit forces a refresh" guarantee docs/architecture.md advertises."""
|
|
12
21
|
try:
|
|
13
22
|
root = os.path.realpath(project_root) if project_root else ""
|
|
14
|
-
|
|
23
|
+
# Relative targets resolve inside the project; absolute ones are already anchored.
|
|
24
|
+
raw = target if os.path.isabs(target) or not root else os.path.join(root, target)
|
|
25
|
+
path = os.path.realpath(raw)
|
|
15
26
|
inside_root = (root and path.startswith(root + os.sep)) or path == root
|
|
16
27
|
if inside_root and os.path.isfile(path):
|
|
17
28
|
with open(path, "rb") as fh:
|
|
@@ -18,6 +18,17 @@ def resolve_root(args: Any) -> str:
|
|
|
18
18
|
return getattr(args, "project_root", None) or os.getcwd()
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
def require_dir(project_root: str, command: str) -> str | None:
|
|
22
|
+
"""An error line when *project_root* is not a directory, else None.
|
|
23
|
+
|
|
24
|
+
A mistyped path used to produce confident, well-formed output about a directory that does not
|
|
25
|
+
exist — `setup /typo` rendered a full three-engine health table for it — which is the worst
|
|
26
|
+
possible response to a typo in a script, because it is indistinguishable from success."""
|
|
27
|
+
if os.path.isdir(project_root):
|
|
28
|
+
return None
|
|
29
|
+
return f"{command} failed: not a directory: {project_root}"
|
|
30
|
+
|
|
31
|
+
|
|
21
32
|
def emit(report: dict, *, as_json: bool, render: Callable[[dict], str]) -> None:
|
|
22
33
|
"""Print a report as structured JSON (--json) or as its human-facing text rendering."""
|
|
23
34
|
if as_json:
|
|
@@ -3,14 +3,21 @@
|
|
|
3
3
|
import json
|
|
4
4
|
from typing import Any
|
|
5
5
|
|
|
6
|
-
from codeintel.commands._common import never_raise, resolve_root
|
|
6
|
+
from codeintel.commands._common import never_raise, require_dir, resolve_root
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
# code=1: this command's job is to WRITE A FILE. Exiting 0 after failing to write it
|
|
10
|
+
# reports success to any `make`/CI step gating on $? while nothing was produced.
|
|
11
|
+
@never_raise("graph failed: {exc}", code=1)
|
|
10
12
|
def run(args: Any) -> int:
|
|
11
13
|
from codeintel import grapher
|
|
12
14
|
|
|
13
|
-
|
|
15
|
+
project_root = resolve_root(args)
|
|
16
|
+
problem = require_dir(project_root, "graph")
|
|
17
|
+
if problem:
|
|
18
|
+
print(problem)
|
|
19
|
+
return 1
|
|
20
|
+
payload = grapher.build_graph_payload(project_root, limit=args.limit)
|
|
14
21
|
nodes, edges = len(payload.get("nodes", [])), len(payload.get("edges", []))
|
|
15
22
|
|
|
16
23
|
if not args.html:
|
|
@@ -12,6 +12,13 @@ def run(args: Any) -> int:
|
|
|
12
12
|
from codeintel.semantic_db import SemanticDb, default_db_path
|
|
13
13
|
|
|
14
14
|
project_root = resolve_root(args)
|
|
15
|
+
# Validate before doing anything. `codeintel index /typo/path` walked nothing, found nothing,
|
|
16
|
+
# and printed "Nothing new to index" at exit 0 — indistinguishable from a correct incremental
|
|
17
|
+
# run, which is the worst possible response to a mistyped path in a script.
|
|
18
|
+
if not os.path.isdir(project_root):
|
|
19
|
+
print(f"index failed: not a directory: {project_root}")
|
|
20
|
+
return 1
|
|
21
|
+
failed = False
|
|
15
22
|
# Wrap the whole semantic pass so a setup failure (e.g. an unresolvable home dir →
|
|
16
23
|
# Path.home() raising) degrades with a message, like every other subcommand, not a traceback.
|
|
17
24
|
try:
|
|
@@ -32,12 +39,18 @@ def run(args: Any) -> int:
|
|
|
32
39
|
).index(project_root)
|
|
33
40
|
if count > 0:
|
|
34
41
|
print(f"Indexed {count} chunks")
|
|
42
|
+
elif count < 0:
|
|
43
|
+
# Indexer.index() returns -1 for an unrecoverable failure. `> 0` sent that into
|
|
44
|
+
# the "Nothing new to index" branch, so a total failure read as a clean no-op.
|
|
45
|
+
print("index failed — the indexer could not complete (see the warnings above)")
|
|
46
|
+
failed = True
|
|
35
47
|
else:
|
|
36
48
|
print("Nothing new to index")
|
|
37
49
|
finally:
|
|
38
50
|
db.close()
|
|
39
51
|
except Exception as exc:
|
|
40
52
|
print(f"index failed: {exc}")
|
|
53
|
+
failed = True
|
|
41
54
|
|
|
42
55
|
# best-effort graph reindex
|
|
43
56
|
import shutil
|
|
@@ -60,4 +73,4 @@ def run(args: Any) -> int:
|
|
|
60
73
|
except Exception:
|
|
61
74
|
pass
|
|
62
75
|
|
|
63
|
-
return 0
|
|
76
|
+
return 1 if failed else 0
|
|
@@ -2,17 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
|
-
from codeintel.commands._common import never_raise, resolve_root
|
|
5
|
+
from codeintel.commands._common import never_raise, require_dir, resolve_root
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
# Never-raise parity with the MCP code.map handler — degrade, don't crash.
|
|
9
|
-
|
|
9
|
+
# code=1: this command's job is to WRITE A FILE. Exiting 0 after failing to write it
|
|
10
|
+
# reports success to any `make`/CI step gating on $? while nothing was produced.
|
|
11
|
+
@never_raise("map failed: {exc}", code=1)
|
|
10
12
|
def run(args: Any) -> int:
|
|
11
13
|
from codeintel.injector import Injector
|
|
12
14
|
from codeintel.mapper import MapGenerator
|
|
13
15
|
from codeintel.providers.graph import GraphProvider
|
|
14
16
|
|
|
15
17
|
project_root = resolve_root(args)
|
|
18
|
+
problem = require_dir(project_root, "map")
|
|
19
|
+
if problem:
|
|
20
|
+
print(problem)
|
|
21
|
+
return 1
|
|
16
22
|
try:
|
|
17
23
|
provider = GraphProvider()
|
|
18
24
|
except Exception:
|
|
@@ -2,16 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
|
-
from codeintel.commands._common import emit, never_raise, resolve_root
|
|
5
|
+
from codeintel.commands._common import emit, never_raise, require_dir, resolve_root
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
@never_raise("setup unavailable: {exc}")
|
|
9
9
|
def run(args: Any) -> int:
|
|
10
10
|
from codeintel import onboarding
|
|
11
11
|
|
|
12
|
+
project_root = resolve_root(args)
|
|
13
|
+
problem = require_dir(project_root, "setup")
|
|
14
|
+
if problem:
|
|
15
|
+
print(problem)
|
|
16
|
+
return 1
|
|
12
17
|
all_steps = getattr(args, "all_steps", False) # --all implies every automatable step
|
|
13
18
|
report = onboarding.run_setup(
|
|
14
|
-
|
|
19
|
+
project_root,
|
|
15
20
|
install_uv=args.install_uv or all_steps,
|
|
16
21
|
install_deps=args.install_deps or all_steps,
|
|
17
22
|
do_index=args.index or all_steps,
|
|
@@ -4,7 +4,7 @@ import datetime
|
|
|
4
4
|
import os
|
|
5
5
|
from typing import Any
|
|
6
6
|
|
|
7
|
-
from codeintel.commands._common import never_raise, resolve_root
|
|
7
|
+
from codeintel.commands._common import never_raise, require_dir, resolve_root
|
|
8
8
|
|
|
9
9
|
# "available" alone was the misleading word: it meant "a binary is on PATH", which is not the same
|
|
10
10
|
# as runnable, and not the same as usable on THIS repo. Say which.
|
|
@@ -16,6 +16,10 @@ def run(args: Any) -> int:
|
|
|
16
16
|
from codeintel import server
|
|
17
17
|
|
|
18
18
|
project_root = resolve_root(args)
|
|
19
|
+
problem = require_dir(project_root, "status")
|
|
20
|
+
if problem:
|
|
21
|
+
print(problem)
|
|
22
|
+
return 1
|
|
19
23
|
status = server.code_status_handler({"project_root": project_root})
|
|
20
24
|
|
|
21
25
|
readiness = status.get("readiness") or {}
|
|
@@ -40,6 +40,15 @@ _UNCACHED_OPS: frozenset[str] = frozenset({"changed", "changes"})
|
|
|
40
40
|
_ADOPTABLE_ENGINES: frozenset[str] = frozenset({"graph", "lsp", "semantic"})
|
|
41
41
|
|
|
42
42
|
|
|
43
|
+
def _mark_reindexing(result: Result, reindexing: bool) -> Result:
|
|
44
|
+
"""Flag an answer served while a reindex for its project is still running."""
|
|
45
|
+
if not reindexing or result.get("result") is None:
|
|
46
|
+
return result
|
|
47
|
+
return {**result, "reindexing": True,
|
|
48
|
+
"hint": "a reindex is in progress — this answer reflects the index as of the last "
|
|
49
|
+
"completed pass; re-ask shortly if you have just changed this code"}
|
|
50
|
+
|
|
51
|
+
|
|
43
52
|
class Gateway:
|
|
44
53
|
def __init__(self, graph=None, lsp=None, semantic=None, policy: TieringPolicy | None = None,
|
|
45
54
|
reindexer: Reindexer | None = None):
|
|
@@ -191,6 +200,15 @@ class Gateway:
|
|
|
191
200
|
except Exception:
|
|
192
201
|
return True
|
|
193
202
|
|
|
203
|
+
def allows_root(self, role: str, project_root: str) -> bool:
|
|
204
|
+
"""Whether *role* may target *project_root*. The companion to ``allows`` — an op gate alone
|
|
205
|
+
leaves the TARGET unbounded, which is how `doctor` and `status` could still be pointed at
|
|
206
|
+
any readable directory after `query` had been scoped."""
|
|
207
|
+
try:
|
|
208
|
+
return self._policy is None or self._policy.is_root_allowed(role, project_root)
|
|
209
|
+
except Exception:
|
|
210
|
+
return True
|
|
211
|
+
|
|
194
212
|
def query(
|
|
195
213
|
self,
|
|
196
214
|
op=None,
|
|
@@ -206,20 +224,37 @@ class Gateway:
|
|
|
206
224
|
engine_str = str(engine or "").strip() or "auto"
|
|
207
225
|
was_auto = engine_str == "auto"
|
|
208
226
|
|
|
209
|
-
# Policy check FIRST — a role denied
|
|
210
|
-
# no
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
227
|
+
# Policy check FIRST — a role denied here does NO work (no reindex, no dispatch, no
|
|
228
|
+
# cache lookup, and critically no on-demand indexing walk). Applies to the modern
|
|
229
|
+
# provider path; the legacy list path has none.
|
|
230
|
+
if self._legacy_providers is None and self._policy is not None:
|
|
231
|
+
if not self._policy.is_allowed(role, op_str):
|
|
232
|
+
return safe_null_result(op_str, target_str, reason="op-not-allowed-for-role")
|
|
233
|
+
# `project_root` arrives in the request body. Without this check any role able to
|
|
234
|
+
# call `search` could name ANY directory the server process can read, and the
|
|
235
|
+
# semantic provider would walk, index, and return its contents — an op allowlist
|
|
236
|
+
# never sees the target. Denied before maybe_reindex, so a rejected path is not
|
|
237
|
+
# even touched.
|
|
238
|
+
if not self._policy.is_root_allowed(role, str(project_root or "")):
|
|
239
|
+
return safe_null_result(op_str, target_str, reason="root-not-allowed-for-role",
|
|
240
|
+
hint="this token's role is not scoped to that "
|
|
241
|
+
"project_root (see the [roots] table in auth.toml)")
|
|
217
242
|
|
|
218
243
|
try:
|
|
219
244
|
self._reindexer.maybe_reindex(str(project_root or ""))
|
|
220
245
|
except Exception:
|
|
221
246
|
pass
|
|
222
247
|
|
|
248
|
+
# If a reindex is running, this answer comes from the PREVIOUS index. Structural
|
|
249
|
+
# answers (callers/impact/hotspots) hash a symbol name, not file bytes, so nothing
|
|
250
|
+
# else in the envelope can reveal that — and an agent that just edited and asked
|
|
251
|
+
# "what did I break?" lands precisely here. Busting the cache would not help: the
|
|
252
|
+
# index itself is behind, so re-asking refetches the same stale data.
|
|
253
|
+
try:
|
|
254
|
+
reindexing = self._reindexer.reindex_pending(str(project_root or ""))
|
|
255
|
+
except Exception:
|
|
256
|
+
reindexing = False
|
|
257
|
+
|
|
223
258
|
# Legacy list-based path (backward compat with pre-Phase-2 tests)
|
|
224
259
|
if self._legacy_providers is not None:
|
|
225
260
|
for p in self._legacy_providers:
|
|
@@ -240,6 +275,15 @@ class Gateway:
|
|
|
240
275
|
if engine_str == "auto":
|
|
241
276
|
engine_str = _AUTO_ENGINE.get(op_str, "graph")
|
|
242
277
|
|
|
278
|
+
# Cache under what was ASKED, not what auto resolved to. `auto` and an explicit
|
|
279
|
+
# `graph` both resolved to "graph" and so shared one key — but they are different
|
|
280
|
+
# questions: `auto` accepts the overview LSP fallback below, an explicit `graph`
|
|
281
|
+
# does not. One `auto` miss therefore parked an LSP answer under the graph key, and
|
|
282
|
+
# the next explicit `engine=graph` request got it back with `cached: true` and an
|
|
283
|
+
# `engine: "lsp"` field contradicting its own request. Reachable on any cold start,
|
|
284
|
+
# since "graph not indexed yet" is the normal first-query state.
|
|
285
|
+
cache_engine = "auto" if was_auto else engine_str
|
|
286
|
+
|
|
243
287
|
root_str = project_root or ""
|
|
244
288
|
|
|
245
289
|
# Freshness token — bumps when a background reindex completes, so a cached
|
|
@@ -259,7 +303,7 @@ class Gateway:
|
|
|
259
303
|
if engine_str in _FANOUT_ENGINES:
|
|
260
304
|
cached_result = (
|
|
261
305
|
None if uncacheable
|
|
262
|
-
else self._cache.get(op_str, target_str,
|
|
306
|
+
else self._cache.get(op_str, target_str, cache_engine, root_str, freshness)
|
|
263
307
|
)
|
|
264
308
|
if cached_result is not None:
|
|
265
309
|
return {**cached_result, "cached": True}
|
|
@@ -268,13 +312,13 @@ class Gateway:
|
|
|
268
312
|
fan_results = self._fan_out(engines, op_str, target_str, budget, project_root)
|
|
269
313
|
result = self._merge(fan_results, op_str, target_str, engine_str)
|
|
270
314
|
if not uncacheable:
|
|
271
|
-
self._cache.put(op_str, target_str,
|
|
315
|
+
self._cache.put(op_str, target_str, cache_engine, root_str, result, freshness)
|
|
272
316
|
return result
|
|
273
317
|
|
|
274
318
|
# Single-engine dispatch (`uncacheable`, computed above, also guards this path).
|
|
275
319
|
cached_result = (
|
|
276
320
|
None if uncacheable
|
|
277
|
-
else self._cache.get(op_str, target_str,
|
|
321
|
+
else self._cache.get(op_str, target_str, cache_engine, root_str, freshness)
|
|
278
322
|
)
|
|
279
323
|
if cached_result is not None:
|
|
280
324
|
return {**cached_result, "cached": True}
|
|
@@ -298,8 +342,8 @@ class Gateway:
|
|
|
298
342
|
result = lsp_result
|
|
299
343
|
|
|
300
344
|
if not uncacheable:
|
|
301
|
-
self._cache.put(op_str, target_str,
|
|
302
|
-
return result
|
|
345
|
+
self._cache.put(op_str, target_str, cache_engine, root_str, result, freshness)
|
|
346
|
+
return _mark_reindexing(result, reindexing)
|
|
303
347
|
|
|
304
348
|
except Exception as exc:
|
|
305
349
|
log_swallowed("Gateway.query", exc)
|
|
@@ -168,8 +168,10 @@ class _Handler(BaseHTTPRequestHandler):
|
|
|
168
168
|
return
|
|
169
169
|
|
|
170
170
|
# Everything else is auth-gated when auth is configured (metrics can reveal usage patterns;
|
|
171
|
-
# status reveals engine/index state).
|
|
172
|
-
|
|
171
|
+
# status reveals engine/index state). The role is kept, not discarded: /code/status takes a
|
|
172
|
+
# project_root, and answering it for any path regardless of the role's [roots] was the same
|
|
173
|
+
# cross-tenant disclosure that scoping /code/query was meant to close.
|
|
174
|
+
ok, role = self._resolve_role()
|
|
173
175
|
if not ok:
|
|
174
176
|
self._send_json(401, {"error": "unauthorized"})
|
|
175
177
|
return
|
|
@@ -185,7 +187,9 @@ class _Handler(BaseHTTPRequestHandler):
|
|
|
185
187
|
return
|
|
186
188
|
if path == "/code/status":
|
|
187
189
|
project_root = (parse_qs(parsed.query).get("project_root") or [""])[0]
|
|
188
|
-
|
|
190
|
+
# role is server-authoritative, exactly as on the POST path — never read from input.
|
|
191
|
+
self._send_json(200, code_status_handler(
|
|
192
|
+
{"project_root": project_root, "role": role}))
|
|
189
193
|
return
|
|
190
194
|
self._send_json(404, {"error": "not-found"})
|
|
191
195
|
|
|
@@ -13,6 +13,15 @@ if TYPE_CHECKING:
|
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
16
|
+
# Hard ceiling on the characters embedded for ONE chunk. Line-based splitting cannot bound a
|
|
17
|
+
# minified or generated single-line file, and the embedder's memory use scales with input size.
|
|
18
|
+
# Generous enough that no hand-written function is affected.
|
|
19
|
+
_MAX_CHUNK_CHARS = 200_000
|
|
20
|
+
|
|
21
|
+
# Bytes examined when deciding whether a file is binary. A NUL in the first block is the classic
|
|
22
|
+
# signal and is what `git` itself uses.
|
|
23
|
+
_BINARY_SNIFF_BYTES = 8192
|
|
24
|
+
|
|
16
25
|
_INDEXED_EXTS = frozenset({
|
|
17
26
|
".py", ".md",
|
|
18
27
|
".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", # TS/JS variants
|
|
@@ -103,6 +112,15 @@ def _ts_decl_is_function(node) -> bool:
|
|
|
103
112
|
return False
|
|
104
113
|
|
|
105
114
|
|
|
115
|
+
def _looks_binary(path) -> bool:
|
|
116
|
+
"""Whether *path* is binary, by the same rule git uses: a NUL byte in the opening block."""
|
|
117
|
+
try:
|
|
118
|
+
with open(path, "rb") as fh:
|
|
119
|
+
return b"\x00" in fh.read(_BINARY_SNIFF_BYTES)
|
|
120
|
+
except OSError:
|
|
121
|
+
return True # unreadable → treat as unindexable rather than crash
|
|
122
|
+
|
|
123
|
+
|
|
106
124
|
class Indexer:
|
|
107
125
|
def __init__(
|
|
108
126
|
self,
|
|
@@ -205,8 +223,26 @@ class Indexer:
|
|
|
205
223
|
except Exception as exc:
|
|
206
224
|
logger.warning("Cleanup pass failed: %s", exc)
|
|
207
225
|
|
|
226
|
+
@staticmethod
|
|
227
|
+
def _binary_check(path) -> bool:
|
|
228
|
+
return _looks_binary(path)
|
|
229
|
+
|
|
208
230
|
def _walk_files(self, root: Path):
|
|
231
|
+
"""Indexable files inside *root* — and strictly inside it.
|
|
232
|
+
|
|
233
|
+
``os.walk`` defaults to ``followlinks=False``, which stops recursion into symlinked
|
|
234
|
+
DIRECTORIES but says nothing about symlinked FILES: those still appear in ``filenames``,
|
|
235
|
+
and the later ``open()`` follows them transparently. That made a planted symlink a
|
|
236
|
+
complete bypass of RBAC project scoping — a tenant able to write inside their own allowed
|
|
237
|
+
root could link to any file the server process can read and have it indexed, embedded, and
|
|
238
|
+
returned as a search snippet. Demonstrated before this guard existed.
|
|
239
|
+
|
|
240
|
+
So every candidate is resolved and required to land back under the resolved root."""
|
|
209
241
|
ignores = set(_SKIP_DIRS) | set(_DEFAULT_IGNORES) | self._load_gitignore(root)
|
|
242
|
+
try:
|
|
243
|
+
real_root = os.path.realpath(root)
|
|
244
|
+
except Exception:
|
|
245
|
+
real_root = str(root)
|
|
210
246
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
211
247
|
dirnames[:] = [
|
|
212
248
|
d for d in dirnames
|
|
@@ -215,8 +251,39 @@ class Indexer:
|
|
|
215
251
|
for fname in filenames:
|
|
216
252
|
if fname in ignores:
|
|
217
253
|
continue
|
|
218
|
-
if Path(fname).suffix.lower() in _INDEXED_EXTS:
|
|
219
|
-
|
|
254
|
+
if Path(fname).suffix.lower() not in _INDEXED_EXTS:
|
|
255
|
+
continue
|
|
256
|
+
candidate = Path(dirpath) / fname
|
|
257
|
+
try:
|
|
258
|
+
real = os.path.realpath(candidate)
|
|
259
|
+
except Exception:
|
|
260
|
+
continue
|
|
261
|
+
if real != real_root and not real.startswith(real_root + os.sep):
|
|
262
|
+
logger.warning("skipping %s — it resolves outside the indexed root", candidate)
|
|
263
|
+
continue
|
|
264
|
+
# A HARDLINK is a second directory entry for the same inode. It is physically
|
|
265
|
+
# inside the root, so its realpath is inside the root and the check above passes
|
|
266
|
+
# — `realpath` cannot see it. That reopened exactly the hole the symlink guard
|
|
267
|
+
# closes: a tenant able to write in their own root could `ln` another tenant's
|
|
268
|
+
# file in and have it indexed. There is no way to ask "does this inode also live
|
|
269
|
+
# outside?", so treat extra links as disqualifying. Measured at 0 occurrences
|
|
270
|
+
# across 3213 source files of a real repo, so the false-positive cost is nil.
|
|
271
|
+
try:
|
|
272
|
+
links = os.stat(real).st_nlink
|
|
273
|
+
except OSError:
|
|
274
|
+
continue
|
|
275
|
+
if links > 1:
|
|
276
|
+
logger.warning("skipping %s — %d hard links, so its content may also live "
|
|
277
|
+
"outside the indexed root", candidate, links)
|
|
278
|
+
continue
|
|
279
|
+
# A source extension is not a promise of source. A compiled artifact or blob named
|
|
280
|
+
# `.py` was read with errors="replace" and embedded as replacement-character
|
|
281
|
+
# garbage — 196KB of /dev/urandom produced 162 chunks — which then competed for
|
|
282
|
+
# rank against real code in every search.
|
|
283
|
+
if _looks_binary(candidate):
|
|
284
|
+
logger.warning("skipping %s — looks binary despite its extension", candidate)
|
|
285
|
+
continue
|
|
286
|
+
yield candidate
|
|
220
287
|
|
|
221
288
|
# ---- chunk-span computation ------------------------------------------------------------
|
|
222
289
|
# A file is turned into a list of 0-based, half-open ``(start, end)`` line spans; every
|
|
@@ -451,6 +518,17 @@ class Indexer:
|
|
|
451
518
|
if not chunk_lines:
|
|
452
519
|
continue
|
|
453
520
|
chunk_text = "".join(chunk_lines)
|
|
521
|
+
# Cap chunk BYTES, not just lines. `_maybe_split` splits on line boundaries, so a
|
|
522
|
+
# minified bundle or a generated one-liner is a single unsplittable chunk however
|
|
523
|
+
# large: a 20MB one-line .py peaked at 3.4GB RSS through the embedder, and a 40MB
|
|
524
|
+
# one extrapolates past 8GB. That runs on the reindexer's daemon thread inside the
|
|
525
|
+
# long-lived MCP server, so it can take the agent host down, not just a CLI run.
|
|
526
|
+
# Truncation is the right trade — the head of a chunk carries its identifying
|
|
527
|
+
# content, and an over-long minified line has no retrieval value past that anyway.
|
|
528
|
+
if len(chunk_text) > _MAX_CHUNK_CHARS:
|
|
529
|
+
logger.warning("truncating an oversized chunk in %s (%d chars) to %d",
|
|
530
|
+
rel_path, len(chunk_text), _MAX_CHUNK_CHARS)
|
|
531
|
+
chunk_text = chunk_text[:_MAX_CHUNK_CHARS]
|
|
454
532
|
if not chunk_text.strip():
|
|
455
533
|
# EC3.4: never embed empty/whitespace-only chunks (zero vectors pollute results).
|
|
456
534
|
chunk_count += 1
|