locusfiles 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- locus/LICENSE +105 -0
- locus/NOTICE +18 -0
- locus/THIRD_PARTY_LICENSES.md +34 -0
- locus/__init__.py +3 -0
- locus/cli.py +421 -0
- locus/config.py +196 -0
- locus/ingest/__init__.py +1 -0
- locus/ingest/chunker.py +103 -0
- locus/ingest/collection.py +38 -0
- locus/ingest/embedder.py +434 -0
- locus/ingest/errors.py +443 -0
- locus/ingest/estimator.py +423 -0
- locus/ingest/indexer.py +342 -0
- locus/ingest/notion_source.py +464 -0
- locus/ingest/parsers.py +93 -0
- locus/ingest/retry.py +193 -0
- locus/ingest/store.py +312 -0
- locus/ingest/watcher.py +342 -0
- locus/mcp_server.py +1728 -0
- locus/preflight.py +200 -0
- locus/relay_signing.py +251 -0
- locusfiles-0.1.0.dist-info/METADATA +91 -0
- locusfiles-0.1.0.dist-info/RECORD +28 -0
- locusfiles-0.1.0.dist-info/WHEEL +4 -0
- locusfiles-0.1.0.dist-info/entry_points.txt +2 -0
- locusfiles-0.1.0.dist-info/licenses/src/locus/LICENSE +105 -0
- locusfiles-0.1.0.dist-info/licenses/src/locus/NOTICE +18 -0
- locusfiles-0.1.0.dist-info/licenses/src/locus/THIRD_PARTY_LICENSES.md +34 -0
locus/LICENSE
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Functional Source License, Version 1.1, ALv2 Future License
|
|
2
|
+
|
|
3
|
+
## Abbreviation
|
|
4
|
+
|
|
5
|
+
FSL-1.1-ALv2
|
|
6
|
+
|
|
7
|
+
## Notice
|
|
8
|
+
|
|
9
|
+
Copyright 2026 Yonathan Levy
|
|
10
|
+
|
|
11
|
+
## Terms and Conditions
|
|
12
|
+
|
|
13
|
+
### Licensor ("We")
|
|
14
|
+
|
|
15
|
+
The party offering the Software under these Terms and Conditions.
|
|
16
|
+
|
|
17
|
+
### The Software
|
|
18
|
+
|
|
19
|
+
The "Software" is each version of the software that we make available under
|
|
20
|
+
these Terms and Conditions, as indicated by our inclusion of these Terms and
|
|
21
|
+
Conditions with the Software.
|
|
22
|
+
|
|
23
|
+
### License Grant
|
|
24
|
+
|
|
25
|
+
Subject to your compliance with this License Grant and the Patents,
|
|
26
|
+
Redistribution and Trademark clauses below, we hereby grant you the right to
|
|
27
|
+
use, copy, modify, create derivative works, publicly perform, publicly display
|
|
28
|
+
and redistribute the Software for any Permitted Purpose identified below.
|
|
29
|
+
|
|
30
|
+
### Permitted Purpose
|
|
31
|
+
|
|
32
|
+
A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
|
|
33
|
+
means making the Software available to others in a commercial product or
|
|
34
|
+
service that:
|
|
35
|
+
|
|
36
|
+
1. substitutes for the Software;
|
|
37
|
+
|
|
38
|
+
2. substitutes for any other product or service we offer using the Software
|
|
39
|
+
that exists as of the date we make the Software available; or
|
|
40
|
+
|
|
41
|
+
3. offers the same or substantially similar functionality as the Software.
|
|
42
|
+
|
|
43
|
+
Permitted Purposes specifically include using the Software:
|
|
44
|
+
|
|
45
|
+
1. for your internal use and access;
|
|
46
|
+
|
|
47
|
+
2. for non-commercial education;
|
|
48
|
+
|
|
49
|
+
3. for non-commercial research; and
|
|
50
|
+
|
|
51
|
+
4. in connection with professional services that you provide to a licensee
|
|
52
|
+
using the Software in accordance with these Terms and Conditions.
|
|
53
|
+
|
|
54
|
+
### Patents
|
|
55
|
+
|
|
56
|
+
To the extent your use for a Permitted Purpose would necessarily infringe our
|
|
57
|
+
patents, the license grant above includes a license under our patents. If you
|
|
58
|
+
make a claim against any party that the Software infringes or contributes to
|
|
59
|
+
the infringement of any patent, then your patent license to the Software ends
|
|
60
|
+
immediately.
|
|
61
|
+
|
|
62
|
+
### Redistribution
|
|
63
|
+
|
|
64
|
+
The Terms and Conditions apply to all copies, modifications and derivatives of
|
|
65
|
+
the Software.
|
|
66
|
+
|
|
67
|
+
If you redistribute any copies, modifications or derivatives of the Software,
|
|
68
|
+
you must include a copy of or a link to these Terms and Conditions and not
|
|
69
|
+
remove any copyright notices provided in or with the Software.
|
|
70
|
+
|
|
71
|
+
### Disclaimer
|
|
72
|
+
|
|
73
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
|
|
74
|
+
IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
|
|
75
|
+
PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
|
|
76
|
+
|
|
77
|
+
IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
|
|
78
|
+
SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
|
|
79
|
+
EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
|
|
80
|
+
|
|
81
|
+
### Trademarks
|
|
82
|
+
|
|
83
|
+
Except for displaying the License Details and identifying us as the origin of
|
|
84
|
+
the Software, you have no right under these Terms and Conditions to use our
|
|
85
|
+
trademarks, trade names, service marks or product names.
|
|
86
|
+
|
|
87
|
+
## Grant of Future License
|
|
88
|
+
|
|
89
|
+
We hereby irrevocably grant you an additional license to use the Software under
|
|
90
|
+
the Apache License, Version 2.0 that is effective on the second anniversary of
|
|
91
|
+
the date we make the Software available. On or after that date, you may use the
|
|
92
|
+
Software under the Apache License, Version 2.0, in which case the following
|
|
93
|
+
will apply:
|
|
94
|
+
|
|
95
|
+
Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
96
|
+
this file except in compliance with the License.
|
|
97
|
+
|
|
98
|
+
You may obtain a copy of the License at
|
|
99
|
+
|
|
100
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
101
|
+
|
|
102
|
+
Unless required by applicable law or agreed to in writing, software distributed
|
|
103
|
+
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
104
|
+
CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
105
|
+
specific language governing permissions and limitations under the License.
|
locus/NOTICE
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Locus engine (the `locusfiles` package)
|
|
2
|
+
Copyright 2026 Yonathan Levy
|
|
3
|
+
|
|
4
|
+
Licensed under the Functional Source License, Version 1.1, ALv2 Future License
|
|
5
|
+
(FSL-1.1-ALv2). The full terms are in the LICENSE file shipped with this package.
|
|
6
|
+
|
|
7
|
+
Third-party software
|
|
8
|
+
--------------------
|
|
9
|
+
This package contains no third-party source code. It depends on third-party
|
|
10
|
+
packages (mcp, chromadb, pydantic, python-dotenv, watchdog, pypdfium2,
|
|
11
|
+
python-docx, and optional extras) that pip, pipx or uv installs separately from
|
|
12
|
+
their own upstream releases. Each keeps its own licence and notices, which ship
|
|
13
|
+
with that package. THIRD_PARTY_LICENSES.md lists every direct dependency, its
|
|
14
|
+
declared licence and where it comes from.
|
|
15
|
+
|
|
16
|
+
With LOCUS_EMBEDDER=local, the engine downloads the BAAI/bge-small-en-v1.5
|
|
17
|
+
embedding model (MIT licence) from Hugging Face on first use. The model is not
|
|
18
|
+
included in this package.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Third-party licences: Locus engine (`locusfiles`)
|
|
2
|
+
|
|
3
|
+
Direct runtime dependencies of this package, generated from each installed package's metadata on 2026-09-26.
|
|
4
|
+
None of them is bundled in this package: pip, pipx or uv installs each one from its own upstream
|
|
5
|
+
release, which carries its own licence text and any NOTICE file. Test and lint tools are not listed
|
|
6
|
+
because they are never installed for users. The Locus engine itself is licensed under
|
|
7
|
+
FSL-1.1-ALv2 (see LICENSE and NOTICE).
|
|
8
|
+
|
|
9
|
+
| Package | Needed for | Version range | Licence (as declared) | Checked version | Ships its own NOTICE | Upstream |
|
|
10
|
+
|---|---|---|---|---|---|---|
|
|
11
|
+
| mcp | core | `>=2.2,<2.3` | MIT | 2.2.0 | no | https://github.com/modelcontextprotocol/python-sdk |
|
|
12
|
+
| chromadb | core | `>=1.5,<1.6` | Apache-2.0 | 1.5.9 | no | https://github.com/chroma-core/chroma |
|
|
13
|
+
| pydantic | core | `>=2.5` | MIT | 2.13.5 | no | https://github.com/pydantic/pydantic |
|
|
14
|
+
| python-dotenv | core | `>=1.0` | BSD-3-Clause | 1.2.3 | no | https://github.com/theskumar/python-dotenv |
|
|
15
|
+
| watchdog | core | `>=4.0` | Apache-2.0 | 6.0.0 | no | https://github.com/gorakhargosh/watchdog/ |
|
|
16
|
+
| pypdfium2 | core | `>=4.30` | BSD-3-Clause, Apache-2.0, dependency licenses | 5.13.0 | no | https://github.com/pypdfium2-team/pypdfium2 |
|
|
17
|
+
| python-docx | core | `>=1.1` | MIT | 1.2.0 | no | https://github.com/python-openxml/python-docx |
|
|
18
|
+
| voyageai | extra `voyage` | `>=0.3` | MIT | 0.5.0 | no | https://pypi.org/project/voyageai/ |
|
|
19
|
+
| openai | extra `openai` | `>=1.30` | Apache-2.0 | 3.19.2 | no | https://github.com/openai/openai-python |
|
|
20
|
+
| sentence-transformers | extra `local` | `>=3.0` | Apache-2.0 | 6.1.0 | yes | https://github.com/huggingface/sentence-transformers/ |
|
|
21
|
+
| lancedb | extra `lancedb` | `>=0.15` | Apache-2.0 | 0.39.0 | no | https://github.com/lancedb/lancedb |
|
|
22
|
+
| realtime | extra `relay` | `>=2.31,<2.32` | MIT | 2.31.0 | no | https://github.com/supabase/supabase-py |
|
|
23
|
+
| httpx | extra `relay` | `>=0.27` | BSD-3-Clause | 0.28.1 | no | https://github.com/encode/httpx |
|
|
24
|
+
| cryptography | extra `relay` | `>=42` | Apache-2.0 OR BSD-3-Clause | 50.0.1 | no | https://github.com/pyca/cryptography/ |
|
|
25
|
+
|
|
26
|
+
Notes:
|
|
27
|
+
|
|
28
|
+
- pypdfium2: Its wheel bundles a PDFium binary; the licences of PDFium and the libraries built into it (abseil, FreeType, ICU, libjpeg-turbo, libpng, libtiff, OpenJPEG, zlib and others) ship in that wheel's licenses/ directory.
|
|
29
|
+
|
|
30
|
+
## Models downloaded at run time (not bundled)
|
|
31
|
+
|
|
32
|
+
| Model | Licence | Source | When |
|
|
33
|
+
|---|---|---|---|
|
|
34
|
+
| BAAI/bge-small-en-v1.5 | MIT | https://huggingface.co/BAAI/bge-small-en-v1.5 | Downloaded from Hugging Face on first use when LOCUS_EMBEDDER=local (the `local` extra). |
|
locus/__init__.py
ADDED
locus/cli.py
ADDED
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
"""Command-line entry point: locus index | estimate | query | serve | watch | sync-notion | stats | doctor."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import shlex
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from .config import Settings, get_settings
|
|
12
|
+
from .ingest.collection import open_index
|
|
13
|
+
from .ingest.embedder import Embedder, input_type_status
|
|
14
|
+
from .ingest.indexer import IndexStats
|
|
15
|
+
from .ingest.store import ChromaStore
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _components(settings: Settings) -> tuple[Embedder, ChromaStore]:
|
|
19
|
+
"""Embedder + collection for a command. A legacy managed collection (RET-002) is announced on
|
|
20
|
+
stderr once per command; it keeps working the way it was built."""
|
|
21
|
+
embedder, store, warning = open_index(settings)
|
|
22
|
+
if warning:
|
|
23
|
+
print(f"locus: warning: {warning}", file=sys.stderr)
|
|
24
|
+
return embedder, store
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _command(*words: str, flags: dict[str, bool]) -> str:
|
|
28
|
+
"""A copy-pasteable command: *words* (shell-quoted) plus each flag that is on, in order. The
|
|
29
|
+
"how to continue" hints rebuild the user's own command this way, so a re-run keeps --prune and,
|
|
30
|
+
for a run confirmed with --yes (scripts, agents), --yes (review F7)."""
|
|
31
|
+
return " ".join([*(shlex.quote(w) for w in words), *(flag for flag, on in flags.items() if on)])
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def cmd_index(args: argparse.Namespace) -> int:
|
|
35
|
+
from .ingest.indexer import index_folder
|
|
36
|
+
|
|
37
|
+
settings = get_settings()
|
|
38
|
+
folder = Path(args.folder)
|
|
39
|
+
if args.dry_run:
|
|
40
|
+
return _dry_run(folder, settings)
|
|
41
|
+
embedder, store = _components(settings)
|
|
42
|
+
rerun = _command("locus", "index", str(folder), flags={"--prune": args.prune, "--yes": True})
|
|
43
|
+
refused = _consent_gate(folder, settings, store, assume_yes=args.yes, rerun=rerun)
|
|
44
|
+
if refused is not None:
|
|
45
|
+
return refused
|
|
46
|
+
resume = _command("locus", "index", str(folder), flags={"--prune": args.prune, "--yes": args.yes})
|
|
47
|
+
stats = index_folder(
|
|
48
|
+
folder,
|
|
49
|
+
embedder,
|
|
50
|
+
store,
|
|
51
|
+
settings,
|
|
52
|
+
prune=args.prune,
|
|
53
|
+
resume=f"To continue, re-run `{resume}` — unchanged files are skipped.",
|
|
54
|
+
)
|
|
55
|
+
return _report(stats)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# Exit codes of commands that index: 0 done, 1 every attempted file failed or the user declined,
|
|
59
|
+
# 2 configuration error (cli.main) or no terminal to ask on and no --yes, 3 stopped early on a
|
|
60
|
+
# budget/billing/key/rate-limit stop — re-run to continue, 130 interrupted.
|
|
61
|
+
EXIT_DECLINED = 1
|
|
62
|
+
EXIT_NEEDS_YES = 2
|
|
63
|
+
EXIT_STOPPED = 3
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _consent_gate(folder: Path, settings: Settings, store: ChromaStore, *, assume_yes: bool, rerun: str) -> int | None:
|
|
67
|
+
"""UX-005: before a hosted embedder receives any chunk text, show what would be sent and where,
|
|
68
|
+
and ask. None = go ahead; otherwise the exit code to return without indexing."""
|
|
69
|
+
from .preflight import DECLINED, NEEDS_YES, build_preflight, confirm, needs_consent
|
|
70
|
+
|
|
71
|
+
if not needs_consent(settings):
|
|
72
|
+
return None
|
|
73
|
+
pre = build_preflight(folder, settings, known_hash=store.file_hash)
|
|
74
|
+
outcome = confirm(pre, assume_yes=assume_yes, rerun=rerun)
|
|
75
|
+
if outcome == NEEDS_YES:
|
|
76
|
+
return EXIT_NEEDS_YES
|
|
77
|
+
if outcome == DECLINED:
|
|
78
|
+
return EXIT_DECLINED
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _dry_run(folder: Path, settings: Settings) -> int:
|
|
83
|
+
"""`locus index --dry-run`: the preflight report for any embedder, without building the embedder
|
|
84
|
+
(a local model load could download it) and without embedding or sending anything."""
|
|
85
|
+
from .ingest.embedder import expected_embedder_name
|
|
86
|
+
from .preflight import build_preflight
|
|
87
|
+
|
|
88
|
+
known_hash = None
|
|
89
|
+
if (settings.data_dir / "chroma").exists():
|
|
90
|
+
# Read-only: an existing collection is checked against the embedder and used to count
|
|
91
|
+
# unchanged files; a missing one is not created (that would stamp this embedder's name on it).
|
|
92
|
+
try:
|
|
93
|
+
store = ChromaStore(settings.data_dir, settings.collection, expected_embedder_name(settings), create=False)
|
|
94
|
+
except LookupError:
|
|
95
|
+
store = None
|
|
96
|
+
known_hash = store.file_hash if store is not None else None
|
|
97
|
+
pre = build_preflight(folder, settings, known_hash=known_hash)
|
|
98
|
+
print(pre.render())
|
|
99
|
+
print("\nDry run: nothing was embedded or sent.")
|
|
100
|
+
return 0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _report(stats: IndexStats) -> int:
|
|
104
|
+
"""Print the run summary and per-file errors. A stop's own message was already logged once
|
|
105
|
+
(stderr) by the indexer; it is not repeated here."""
|
|
106
|
+
print(stats.summary())
|
|
107
|
+
for err in stats.errors:
|
|
108
|
+
print(f" ! {err}")
|
|
109
|
+
if stats.stop is not None:
|
|
110
|
+
return EXIT_STOPPED
|
|
111
|
+
return 1 if stats.errors and stats.files_indexed == 0 else 0
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def cmd_estimate(args: argparse.Namespace) -> int:
|
|
115
|
+
"""PROD-012: files, bytes by type and estimated tokens for a folder. Builds no embedder and
|
|
116
|
+
opens no connection (the local embedder alone would download a model)."""
|
|
117
|
+
import json
|
|
118
|
+
|
|
119
|
+
from .ingest.estimator import estimate, render_estimate, scan
|
|
120
|
+
|
|
121
|
+
settings = get_settings()
|
|
122
|
+
result = scan(Path(args.folder), settings)
|
|
123
|
+
est = estimate(result.files, settings, sample_seconds=args.sample_seconds)
|
|
124
|
+
if args.json:
|
|
125
|
+
payload = {"root": str(result.root), "excluded": dict(result.excluded), **est.to_dict()}
|
|
126
|
+
print(json.dumps(payload, indent=2))
|
|
127
|
+
else:
|
|
128
|
+
print(render_estimate(result, est, settings))
|
|
129
|
+
return 0
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def cmd_query(args: argparse.Namespace) -> int:
|
|
133
|
+
settings = get_settings()
|
|
134
|
+
embedder, store = _components(settings)
|
|
135
|
+
hits = store.query(embedder.embed_query(args.query), k=args.k, path_prefix=args.prefix)
|
|
136
|
+
if not hits:
|
|
137
|
+
print("no results (is anything indexed? try `locus stats`)")
|
|
138
|
+
return 1
|
|
139
|
+
for i, hit in enumerate(hits, 1):
|
|
140
|
+
snippet = hit.text.replace("\n", " ")
|
|
141
|
+
snippet = snippet if len(snippet) <= args.width else snippet[: args.width] + "…"
|
|
142
|
+
print(f"{i}. [{hit.score:.3f}] {hit.path} #{hit.chunk_index}\n {snippet}")
|
|
143
|
+
return 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def cmd_serve(args: argparse.Namespace) -> int:
|
|
147
|
+
from .mcp_server import main as serve
|
|
148
|
+
|
|
149
|
+
if args.transport == "relay":
|
|
150
|
+
# The relay worker calls Locus's hosted token endpoint: an invalid LOCUS_API_BASE is a
|
|
151
|
+
# configuration error here (exit 2), while stdio / streamable-http never need it.
|
|
152
|
+
settings = get_settings()
|
|
153
|
+
settings.hosted_url(settings.mcp_relay_token_url)
|
|
154
|
+
serve(args.transport, host=args.host, port=args.port, path=args.path)
|
|
155
|
+
return 0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def cmd_watch(args: argparse.Namespace) -> int:
|
|
159
|
+
from .ingest.indexer import index_folder
|
|
160
|
+
from .ingest.watcher import PendingQueue, watch_folder
|
|
161
|
+
|
|
162
|
+
settings = get_settings()
|
|
163
|
+
embedder, store = _components(settings)
|
|
164
|
+
root = Path(args.folder)
|
|
165
|
+
rerun = _command("locus", "watch", str(root), flags={"--no-initial": args.no_initial, "--yes": True})
|
|
166
|
+
refused = _consent_gate(root, settings, store, assume_yes=args.yes, rerun=rerun)
|
|
167
|
+
if refused is not None:
|
|
168
|
+
return refused
|
|
169
|
+
pending = PendingQueue()
|
|
170
|
+
if not args.no_initial:
|
|
171
|
+
stats = index_folder(
|
|
172
|
+
root,
|
|
173
|
+
embedder,
|
|
174
|
+
store,
|
|
175
|
+
settings,
|
|
176
|
+
prune=True,
|
|
177
|
+
resume="The watcher keeps running, queues them and retries when the pause ends.",
|
|
178
|
+
)
|
|
179
|
+
if stats.stop is not None:
|
|
180
|
+
# RET-008: the files the initial pass did not reach wait in the watcher's queue and are
|
|
181
|
+
# retried when the pause ends, instead of waiting for their next edit.
|
|
182
|
+
pending.hold(stats.pending_paths, stats.stop.error)
|
|
183
|
+
watch_folder(root, embedder, store, settings, pending=pending)
|
|
184
|
+
return 0
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def cmd_sync_notion(args: argparse.Namespace) -> int:
|
|
188
|
+
from .ingest.indexer import index_folder
|
|
189
|
+
from .ingest.notion_source import sync_notion
|
|
190
|
+
|
|
191
|
+
settings = get_settings()
|
|
192
|
+
sync_stats = sync_notion(settings)
|
|
193
|
+
print(f"notion sync: {sync_stats.summary()}")
|
|
194
|
+
|
|
195
|
+
notion_folder = settings.data_dir / "sources" / "notion"
|
|
196
|
+
embedder, store = _components(settings)
|
|
197
|
+
refused = _consent_gate(notion_folder, settings, store, assume_yes=args.yes, rerun="locus sync-notion --yes")
|
|
198
|
+
if refused is not None:
|
|
199
|
+
return refused
|
|
200
|
+
stats = index_folder(
|
|
201
|
+
notion_folder,
|
|
202
|
+
embedder,
|
|
203
|
+
store,
|
|
204
|
+
settings,
|
|
205
|
+
prune=True,
|
|
206
|
+
resume=f"To continue, re-run `{_command('locus', 'sync-notion', flags={'--yes': args.yes})}` — "
|
|
207
|
+
"unchanged pages are skipped.",
|
|
208
|
+
)
|
|
209
|
+
return _report(stats)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def cmd_stats(_: argparse.Namespace) -> int:
|
|
213
|
+
settings = get_settings()
|
|
214
|
+
embedder, store = _components(settings)
|
|
215
|
+
paths = store.list_paths()
|
|
216
|
+
print(f"data_dir : {settings.data_dir}")
|
|
217
|
+
print(f"collection : {settings.collection}")
|
|
218
|
+
print(f"embedder : {embedder.name}")
|
|
219
|
+
print(f"input_type : {input_type_status(embedder)}")
|
|
220
|
+
print(f"files : {len(paths)}")
|
|
221
|
+
print(f"chunks : {store.count()}")
|
|
222
|
+
return 0
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def cmd_doctor(_: argparse.Namespace) -> int:
|
|
226
|
+
"""Diagnose the most common setup failures before they turn into a support message."""
|
|
227
|
+
ok = True
|
|
228
|
+
|
|
229
|
+
def check(label: str, passed: bool, detail: str = "") -> None:
|
|
230
|
+
nonlocal ok
|
|
231
|
+
ok = ok and passed
|
|
232
|
+
mark = "OK " if passed else "FAIL"
|
|
233
|
+
print(f"[{mark}] {label}" + (f" — {detail}" if detail else ""))
|
|
234
|
+
|
|
235
|
+
check("Python interpreter", sys.version_info >= (3, 11), sys.executable)
|
|
236
|
+
for mod in ("chromadb", "mcp", "pydantic", "dotenv", "watchdog", "pypdfium2", "docx"):
|
|
237
|
+
try:
|
|
238
|
+
__import__(mod)
|
|
239
|
+
check(f"dependency: {mod}", True)
|
|
240
|
+
except ImportError as exc:
|
|
241
|
+
check(f"dependency: {mod}", False, f"not importable ({exc}) — re-run pip install -e '.[dev]'")
|
|
242
|
+
|
|
243
|
+
try:
|
|
244
|
+
settings = get_settings()
|
|
245
|
+
except Exception as exc: # noqa: BLE001 - config parsing itself is part of what we diagnose
|
|
246
|
+
check("settings load from .env / environment", False, f"{type(exc).__name__}: {exc}")
|
|
247
|
+
return 1
|
|
248
|
+
check("settings load from .env / environment", True, f"embedder={settings.embedder}")
|
|
249
|
+
if settings.api_base_error:
|
|
250
|
+
# Local commands keep working, but managed embedding and the MCP relay would refuse to start.
|
|
251
|
+
check("LOCUS_API_BASE", False, settings.api_base_error)
|
|
252
|
+
|
|
253
|
+
if settings.embedder == "voyage":
|
|
254
|
+
check("VOYAGE_API_KEY set", bool(_os_env("VOYAGE_API_KEY")))
|
|
255
|
+
elif settings.embedder == "openai":
|
|
256
|
+
check("OPENAI_API_KEY set", bool(_os_env("OPENAI_API_KEY")))
|
|
257
|
+
elif settings.embedder == "local":
|
|
258
|
+
try:
|
|
259
|
+
import sentence_transformers # noqa: F401
|
|
260
|
+
|
|
261
|
+
check("dependency: sentence-transformers (LOCUS_EMBEDDER=local)", True)
|
|
262
|
+
except ImportError:
|
|
263
|
+
check(
|
|
264
|
+
"dependency: sentence-transformers (LOCUS_EMBEDDER=local)",
|
|
265
|
+
False,
|
|
266
|
+
"re-run: pip install -e '.[dev,local]'",
|
|
267
|
+
)
|
|
268
|
+
elif settings.embedder == "managed":
|
|
269
|
+
check(
|
|
270
|
+
"LOCUS_API_KEY set (LOCUS_EMBEDDER=managed)",
|
|
271
|
+
bool(settings.managed_api_key),
|
|
272
|
+
f"relay: {settings.managed_relay_url}",
|
|
273
|
+
)
|
|
274
|
+
elif settings.embedder != "hash":
|
|
275
|
+
check("LOCUS_EMBEDDER value", False, f"{settings.embedder!r} is not voyage/openai/local/managed/hash")
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
settings.data_dir.mkdir(parents=True, exist_ok=True)
|
|
279
|
+
os.chmod(settings.data_dir, 0o700) # see store.py's ChromaStore.__init__ for why
|
|
280
|
+
probe = settings.data_dir / ".doctor_probe"
|
|
281
|
+
probe.write_text("ok")
|
|
282
|
+
probe.unlink()
|
|
283
|
+
check("data directory is writable", True, str(settings.data_dir))
|
|
284
|
+
except OSError as exc:
|
|
285
|
+
check("data directory is writable", False, f"{settings.data_dir}: {exc}")
|
|
286
|
+
|
|
287
|
+
if ok:
|
|
288
|
+
legacy: str | None = None
|
|
289
|
+
try:
|
|
290
|
+
embedder, store, legacy = open_index(settings)
|
|
291
|
+
_ = embedder.embed_query("doctor check")
|
|
292
|
+
store.count()
|
|
293
|
+
check("embedder + store round-trip", True)
|
|
294
|
+
except Exception as exc: # noqa: BLE001 - report, don't crash the diagnostic
|
|
295
|
+
check("embedder + store round-trip", False, f"{type(exc).__name__}: {exc}")
|
|
296
|
+
if legacy:
|
|
297
|
+
# RET-002: not a failure (search works as it always did), but worth acting on.
|
|
298
|
+
print(f"[WARN] collection input_type policy — {legacy}")
|
|
299
|
+
|
|
300
|
+
if ok:
|
|
301
|
+
print("\nAll checks passed — ready for `locus index <folder>`.")
|
|
302
|
+
else:
|
|
303
|
+
print("\nFix the FAIL lines above, then re-run `locus doctor`.")
|
|
304
|
+
return 0 if ok else 1
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _os_env(name: str) -> str:
|
|
308
|
+
import os
|
|
309
|
+
|
|
310
|
+
return os.getenv(name, "")
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
314
|
+
parser = argparse.ArgumentParser(prog="locus", description="Local-first personal AI brain over MCP.")
|
|
315
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
316
|
+
|
|
317
|
+
yes_help = (
|
|
318
|
+
"confirm sending chunk text to the hosted embedder (voyage/openai/managed) without asking; "
|
|
319
|
+
"required when not run from a terminal"
|
|
320
|
+
)
|
|
321
|
+
p = sub.add_parser(
|
|
322
|
+
"index",
|
|
323
|
+
help="index (or re-index) a folder",
|
|
324
|
+
description="Index a folder. With a hosted embedder it first shows what would be sent and where, "
|
|
325
|
+
"and asks. Exit codes: 0 done, 1 all files failed or declined, 2 configuration error or no "
|
|
326
|
+
"--yes without a terminal, 3 stopped early on a budget/billing/key/rate limit (re-run to continue).",
|
|
327
|
+
)
|
|
328
|
+
p.add_argument("folder")
|
|
329
|
+
p.add_argument("--prune", action="store_true", help="remove files that no longer exist")
|
|
330
|
+
p.add_argument("-y", "--yes", action="store_true", help=yes_help)
|
|
331
|
+
p.add_argument(
|
|
332
|
+
"--dry-run",
|
|
333
|
+
action="store_true",
|
|
334
|
+
help="only show what would be indexed and sent (files, types, exclusions, token estimate); send nothing",
|
|
335
|
+
)
|
|
336
|
+
p.set_defaults(func=cmd_index)
|
|
337
|
+
|
|
338
|
+
p = sub.add_parser(
|
|
339
|
+
"estimate",
|
|
340
|
+
help="estimate files, size and embedding tokens for a folder before indexing (sends nothing)",
|
|
341
|
+
)
|
|
342
|
+
p.add_argument("folder")
|
|
343
|
+
p.add_argument("--json", action="store_true", help="machine-readable output")
|
|
344
|
+
p.add_argument(
|
|
345
|
+
"--sample-seconds",
|
|
346
|
+
type=float,
|
|
347
|
+
default=60.0,
|
|
348
|
+
help="time limit for parsing a sample of files to measure them (default 60)",
|
|
349
|
+
)
|
|
350
|
+
p.set_defaults(func=cmd_estimate)
|
|
351
|
+
|
|
352
|
+
p = sub.add_parser("query", help="semantic search from the terminal")
|
|
353
|
+
p.add_argument("query")
|
|
354
|
+
p.add_argument("-k", type=int, default=5)
|
|
355
|
+
p.add_argument("--prefix", default=None, help="restrict to an absolute path prefix")
|
|
356
|
+
p.add_argument("--width", type=int, default=200, help="snippet width")
|
|
357
|
+
p.set_defaults(func=cmd_query)
|
|
358
|
+
|
|
359
|
+
p = sub.add_parser(
|
|
360
|
+
"serve",
|
|
361
|
+
help="run the MCP server on stdio (or streamable-http for a tunnel, or relay for Claude.ai web/ChatGPT)",
|
|
362
|
+
)
|
|
363
|
+
p.add_argument(
|
|
364
|
+
"--transport",
|
|
365
|
+
choices=["stdio", "streamable-http", "relay"],
|
|
366
|
+
default="stdio",
|
|
367
|
+
help="stdio (default, for local MCP clients like Claude Desktop), streamable-http "
|
|
368
|
+
"(for a user-run tunnel to Claude.ai web / ChatGPT — requires LOCUS_MCP_TOKEN in .env), "
|
|
369
|
+
"or relay (Locus-hosted, outbound-only, for Claude.ai web / ChatGPT — requires "
|
|
370
|
+
"LOCUS_API_KEY and the `relay` extra; one computer per account serves at a time and the newest "
|
|
371
|
+
"one takes over. Relay exit codes: 1 = channel lost or token not refreshed (safe to restart), "
|
|
372
|
+
"2 = key rejected/revoked, subscription inactive or channel refused at start, 3 = superseded by "
|
|
373
|
+
"another computer — do "
|
|
374
|
+
"not auto-restart on 3)",
|
|
375
|
+
)
|
|
376
|
+
p.add_argument("--host", default="127.0.0.1", help="bind host for streamable-http (default: localhost only)")
|
|
377
|
+
p.add_argument("--port", type=int, default=8000, help="bind port for streamable-http")
|
|
378
|
+
p.add_argument("--path", default="/mcp", help="URL path for streamable-http")
|
|
379
|
+
p.set_defaults(func=cmd_serve)
|
|
380
|
+
|
|
381
|
+
p = sub.add_parser("watch", help="index a folder, then keep it in sync")
|
|
382
|
+
p.add_argument("folder")
|
|
383
|
+
p.add_argument("--no-initial", action="store_true", help="skip the initial full index")
|
|
384
|
+
p.add_argument("-y", "--yes", action="store_true", help=yes_help)
|
|
385
|
+
p.set_defaults(func=cmd_watch)
|
|
386
|
+
|
|
387
|
+
p = sub.add_parser(
|
|
388
|
+
"sync-notion",
|
|
389
|
+
help="OAuth-authenticate with Notion (first run only), pull page content, and index it",
|
|
390
|
+
)
|
|
391
|
+
p.add_argument("-y", "--yes", action="store_true", help=yes_help)
|
|
392
|
+
p.set_defaults(func=cmd_sync_notion)
|
|
393
|
+
|
|
394
|
+
p = sub.add_parser("stats", help="show index statistics")
|
|
395
|
+
p.set_defaults(func=cmd_stats)
|
|
396
|
+
|
|
397
|
+
p = sub.add_parser("doctor", help="diagnose common setup problems")
|
|
398
|
+
p.set_defaults(func=cmd_doctor)
|
|
399
|
+
return parser
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def main(argv: list[str] | None = None) -> None:
|
|
403
|
+
args = build_parser().parse_args(argv)
|
|
404
|
+
try:
|
|
405
|
+
sys.exit(args.func(args))
|
|
406
|
+
except RuntimeError as exc: # configuration errors (missing key, embedder mismatch)
|
|
407
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
408
|
+
sys.exit(2)
|
|
409
|
+
except (NotADirectoryError, FileNotFoundError) as exc:
|
|
410
|
+
print(f"error: no such folder: {exc}", file=sys.stderr)
|
|
411
|
+
sys.exit(2)
|
|
412
|
+
except OSError as exc: # permission errors, disk issues, etc. — no raw traceback
|
|
413
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
414
|
+
sys.exit(2)
|
|
415
|
+
except KeyboardInterrupt: # Ctrl-C: stop cleanly; a file is only replaced once fully embedded
|
|
416
|
+
print("\ninterrupted — anything already indexed is kept; re-run to continue.", file=sys.stderr)
|
|
417
|
+
sys.exit(130)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
if __name__ == "__main__":
|
|
421
|
+
main()
|