tomefile 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tomefile/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """Local RAG as a single file."""
2
+
3
+ __version__ = "0.1.0"
tomefile/archive.py ADDED
@@ -0,0 +1,93 @@
1
+ from __future__ import annotations
2
+
3
+ import zipfile
4
+ from pathlib import Path
5
+
6
+ from tomefile.errors import TomeError
7
+ from tomefile.hashing import sha256_bytes, sha256_file
8
+ from tomefile.manifest import SHA_RE
9
+
10
+ CHECKSUMS = "checksums.sha256"
11
+ SIGNATURE = "signature.json"
12
+ EXCLUDED = {CHECKSUMS, SIGNATURE}
13
+
14
+
15
+ def checksum_document(members: dict[str, bytes]) -> bytes:
16
+ lines: list[str] = []
17
+ names = sorted((name for name in members if name not in EXCLUDED), key=lambda name: name.encode("utf-8"))
18
+ for name in names:
19
+ lines.append(f"{sha256_bytes(members[name])} {name}")
20
+ return ("\n".join(lines) + "\n").encode("utf-8")
21
+
22
+
23
+ def parse_checksums(raw: bytes) -> dict[str, str]:
24
+ try:
25
+ text = raw.decode("utf-8")
26
+ except UnicodeDecodeError as exc:
27
+ raise TomeError("checksums.sha256 is not UTF-8") from exc
28
+ listed: dict[str, str] = {}
29
+ for number, line in enumerate(text.splitlines(), start=1):
30
+ if line == "":
31
+ continue
32
+ digest, separator, name = line.partition(" ") # sha256sum: digest, two spaces, name
33
+ if separator != " " or SHA_RE.fullmatch(digest) is None or not _safe_name(name):
34
+ raise TomeError(f"checksums.sha256 line {number} is malformed")
35
+ if name in listed:
36
+ raise TomeError(f"checksums.sha256 lists {name} twice")
37
+ listed[name] = digest
38
+ return listed
39
+
40
+
41
+ def verify_checksums(members: dict[str, bytes]) -> bytes:
42
+ raw = members.get(CHECKSUMS)
43
+ if raw is None:
44
+ raise TomeError("checksums.sha256 is missing")
45
+ listed = parse_checksums(raw)
46
+ payload = {name: data for name, data in members.items() if name not in EXCLUDED}
47
+ if set(listed) != set(payload):
48
+ raise TomeError("checksums.sha256 does not match the archive members")
49
+ for name, digest in listed.items():
50
+ if sha256_bytes(payload[name]) != digest:
51
+ raise TomeError(f"checksum mismatch for {name}")
52
+ return raw
53
+
54
+
55
+ def write_archive(path: Path, members: dict[str, bytes]) -> None:
56
+ path.parent.mkdir(parents=True, exist_ok=True)
57
+ with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
58
+ for name in sorted(members, key=lambda item: item.encode("utf-8")):
59
+ info = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0)) # stable hashes
60
+ info.compress_type = zipfile.ZIP_STORED
61
+ info.flag_bits |= 0x800 # UTF-8 names
62
+ info.external_attr = 0o644 << 16
63
+ archive.writestr(info, members[name])
64
+
65
+
66
+ def read_archive(path: Path) -> tuple[str, dict[str, bytes]]:
67
+ digest = sha256_file(path)
68
+ try:
69
+ archive = zipfile.ZipFile(path)
70
+ except zipfile.BadZipFile as exc:
71
+ raise TomeError(f"{path} is not a ZIP archive") from exc
72
+ members: dict[str, bytes] = {}
73
+ with archive:
74
+ for info in archive.infolist():
75
+ if info.is_dir():
76
+ continue
77
+ if info.compress_type != zipfile.ZIP_STORED:
78
+ raise TomeError(f"{info.filename} is compressed; members must use store method")
79
+ if not _safe_name(info.filename):
80
+ raise TomeError(f"unsafe member {info.filename}")
81
+ if info.filename in members:
82
+ raise TomeError(f"duplicate member {info.filename}")
83
+ members[info.filename] = archive.read(info)
84
+ if not members:
85
+ raise TomeError(f"{path} is empty")
86
+ return digest, members
87
+
88
+
89
+ def _safe_name(name: str) -> bool:
90
+ if name == "" or name.startswith("/") or name.endswith("/"):
91
+ return False
92
+ parts = name.split("/")
93
+ return ".." not in parts and "." not in parts and "" not in parts
tomefile/build.py ADDED
@@ -0,0 +1,143 @@
1
+ from __future__ import annotations
2
+
3
+ import tempfile
4
+ from collections.abc import Callable
5
+ from pathlib import Path
6
+
7
+ from tomefile import __version__
8
+ from tomefile.archive import CHECKSUMS, SIGNATURE, checksum_document, write_archive
9
+ from tomefile.chunking import Chunk, chunk_documents, discover_documents
10
+ from tomefile.database import create_database
11
+ from tomefile.errors import TomeError
12
+ from tomefile.hashing import sha256_file
13
+ from tomefile.manifest import (
14
+ CHUNK_SIZE,
15
+ TOP_K,
16
+ LicenseGrant,
17
+ Manifest,
18
+ ModelRef,
19
+ default_name,
20
+ default_signer,
21
+ require_id,
22
+ )
23
+ from tomefile.models import cache_model_file, cached_model
24
+ from tomefile.signing import sign_checksums
25
+
26
+
27
+ def build_tome(
28
+ input_dir: Path,
29
+ output: Path,
30
+ *,
31
+ package_id: str,
32
+ revision: int,
33
+ model: Path,
34
+ embed_id: str,
35
+ embed_license_name: str,
36
+ embed: Callable[[str], list[float]],
37
+ name: str | None = None,
38
+ knowledge_license: str = "internal",
39
+ knowledge_commercial: bool = False,
40
+ knowledge_redistribute: bool = False,
41
+ attribution: bool = True,
42
+ redistribute_embed: bool = True,
43
+ embed_model: bool = False,
44
+ recommended_llm: str = "",
45
+ sign_key: Path | None = None,
46
+ signer: str | None = None,
47
+ cache_root: Path,
48
+ ) -> None:
49
+ package_id = require_id(package_id, "id")
50
+ if revision < 1:
51
+ raise TomeError("revision must be an integer >= 1")
52
+ if output.suffix.lower() != ".tome":
53
+ raise TomeError("output must end in .tome")
54
+ if not embed_id:
55
+ raise TomeError("--embed-id must not be empty")
56
+ if not embed_license_name:
57
+ raise TomeError("--embed-license must not be empty")
58
+ if embed_model and not redistribute_embed:
59
+ raise TomeError("--embed-model requires a redistributable embedding license")
60
+ documents = discover_documents(input_dir)
61
+ chunks = chunk_documents(documents, CHUNK_SIZE)
62
+ if not chunks:
63
+ raise TomeError("no text to index")
64
+ vectors = [_embed_chunk(embed, chunk) for chunk in chunks]
65
+ dimension = len(vectors[0])
66
+ digest = sha256_file(model)
67
+ already_cached = cached_model(cache_root, digest)
68
+ if not redistribute_embed and already_cached is None:
69
+ raise TomeError(
70
+ f"embedding model is not redistributable and {digest} is not in the model cache"
71
+ )
72
+ include_model = redistribute_embed and (embed_model or already_cached is None)
73
+ if already_cached is None and redistribute_embed:
74
+ cache_model_file(model, cache_root)
75
+ manifest = Manifest(
76
+ type="data_tome",
77
+ id=package_id,
78
+ revision=revision,
79
+ name=name or default_name(package_id),
80
+ embed="models/embed.gguf" if include_model else ModelRef(digest, model.name),
81
+ chat=None,
82
+ embedding_id=embed_id,
83
+ embedding_sha256=digest,
84
+ dimensions=dimension,
85
+ recommended_llm=recommended_llm,
86
+ builder="tomefile",
87
+ builder_version=__version__,
88
+ chunker="fixed-size",
89
+ knowledge=LicenseGrant(
90
+ knowledge_license or "internal",
91
+ knowledge_commercial,
92
+ knowledge_redistribute,
93
+ attribution,
94
+ ),
95
+ embed_license=LicenseGrant(embed_license_name, True, redistribute_embed, attribution),
96
+ chat_license=None,
97
+ chunk_size=CHUNK_SIZE,
98
+ retrieve_top_k=TOP_K,
99
+ )
100
+ with tempfile.TemporaryDirectory() as temporary:
101
+ database_path = Path(temporary) / "vectors.db"
102
+ create_database(
103
+ database_path,
104
+ chunks,
105
+ vectors,
106
+ {
107
+ "package_id": package_id,
108
+ "revision": str(revision),
109
+ "metric": "cosine",
110
+ "embedding_id": embed_id,
111
+ "embedding_sha256": digest,
112
+ },
113
+ )
114
+ members: dict[str, bytes] = {
115
+ "manifest.json": manifest.dumps(),
116
+ "knowledge/vectors.db": database_path.read_bytes(),
117
+ }
118
+ if include_model:
119
+ members["models/embed.gguf"] = model.read_bytes()
120
+ _finish_archive(output, members, sign_key, signer or default_signer(package_id))
121
+
122
+
123
+ def _embed_chunk(embed: Callable[[str], list[float]], chunk: Chunk) -> list[float]:
124
+ try:
125
+ vector = embed(chunk.text)
126
+ except Exception as exc:
127
+ raise TomeError(f"embedder rejected {chunk.source}: {exc}") from exc
128
+ if not vector:
129
+ raise TomeError(f"embedder rejected {chunk.source}: empty vector")
130
+ return [float(value) for value in vector]
131
+
132
+
133
+ def _finish_archive(
134
+ output: Path,
135
+ members: dict[str, bytes],
136
+ sign_key: Path | None,
137
+ signer: str,
138
+ ) -> None:
139
+ checksums = checksum_document(members)
140
+ members[CHECKSUMS] = checksums
141
+ if sign_key is not None:
142
+ members[SIGNATURE] = sign_checksums(checksums, sign_key, signer)
143
+ write_archive(output, members)
tomefile/chunking.py ADDED
@@ -0,0 +1,66 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from pathlib import Path
5
+
6
+ from tomefile.errors import TomeError
7
+
8
+ TEXT_SUFFIXES = {".txt", ".md"}
9
+
10
+
11
+ @dataclass(frozen=True)
12
+ class Chunk:
13
+ id: int
14
+ text: str
15
+ source: str
16
+ start: int
17
+ end: int
18
+
19
+
20
+ def discover_documents(input_dir: Path) -> list[tuple[str, str]]:
21
+ found: list[tuple[str, str]] = []
22
+ for path in input_dir.rglob("*"):
23
+ if not path.is_file() or path.suffix.lower() not in TEXT_SUFFIXES:
24
+ continue
25
+ relative = path.relative_to(input_dir).as_posix()
26
+ try:
27
+ text = path.read_text(encoding="utf-8")
28
+ except UnicodeDecodeError as exc:
29
+ raise TomeError(f"{relative} is not valid UTF-8") from exc
30
+ found.append((relative, text))
31
+ found.sort(key=lambda item: item[0].encode("utf-8"))
32
+ return found
33
+
34
+
35
+ def chunk_text(text: str, size: int) -> list[tuple[int, int, str]]:
36
+ if size < 1:
37
+ raise TomeError("chunk size must be positive")
38
+ pieces: list[tuple[int, int, str]] = []
39
+ index = 0
40
+ length = len(text)
41
+ while index < length:
42
+ end = min(index + size, length)
43
+ if end < length:
44
+ # last whitespace in the window; keep it in this chunk
45
+ window = text[index:end]
46
+ cut = -1
47
+ for offset, char in enumerate(window):
48
+ if char.isspace():
49
+ cut = offset
50
+ if cut >= 0:
51
+ end = index + cut + 1
52
+ if end <= index:
53
+ end = index + 1
54
+ pieces.append((index, end, text[index:end]))
55
+ index = end
56
+ return pieces
57
+
58
+
59
+ def chunk_documents(documents: list[tuple[str, str]], size: int) -> list[Chunk]:
60
+ chunks: list[Chunk] = []
61
+ next_id = 1
62
+ for source, text in documents:
63
+ for start, end, piece in chunk_text(text, size):
64
+ chunks.append(Chunk(next_id, piece, source, start, end))
65
+ next_id += 1
66
+ return chunks
tomefile/cli.py ADDED
@@ -0,0 +1,261 @@
1
+ from __future__ import annotations
2
+
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ import typer
7
+ import uvicorn
8
+
9
+ from tomefile import __version__
10
+ from tomefile.errors import TomeError
11
+ from tomefile.opening import prepare
12
+ from tomefile.runtime import HttpUpstream, LlamaServer, check_ollama, choose_port
13
+
14
+ app = typer.Typer(no_args_is_help=True, add_completion=False)
15
+
16
+
17
+ def _show_version(value: bool) -> None:
18
+ if value:
19
+ typer.echo(__version__)
20
+ raise typer.Exit()
21
+
22
+
23
+ @app.callback()
24
+ def _root(
25
+ version: bool = typer.Option(
26
+ False,
27
+ "--version",
28
+ callback=_show_version,
29
+ is_eager=True,
30
+ help="Show version and exit.",
31
+ ),
32
+ ) -> None:
33
+ pass
34
+
35
+
36
+ def _cache_root() -> Path:
37
+ return Path.home() / ".cache" / "tomefile"
38
+
39
+
40
+ def _config_root() -> Path:
41
+ return Path.home() / ".config" / "tomefile"
42
+
43
+
44
+ class _LazyEmbedder:
45
+ # llama-server starts on the first embed(), so flag errors fail first
46
+ def __init__(self, model: Path) -> None:
47
+ self.model = model
48
+ self.server: LlamaServer | None = None
49
+
50
+ def __call__(self, text: str) -> list[float]:
51
+ if self.server is None:
52
+ self.server = LlamaServer(self.model, embeddings=True)
53
+ self.server.start()
54
+ return self.server.embed(text)
55
+
56
+ def stop(self) -> None:
57
+ if self.server is not None:
58
+ self.server.stop()
59
+
60
+
61
+ def _fail(exc: TomeError) -> None:
62
+ typer.echo(str(exc), err=True)
63
+ raise typer.Exit(code=1)
64
+
65
+
66
+ @app.command()
67
+ def build(
68
+ input_dir: Path = typer.Argument(..., exists=True, file_okay=False, dir_okay=True),
69
+ package_id: str = typer.Option(..., "--id"),
70
+ revision: int = typer.Option(..., "--revision"),
71
+ model: Path = typer.Option(..., "--model", exists=True, dir_okay=False),
72
+ embed_id: str = typer.Option(..., "--embed-id"),
73
+ embed_license: str = typer.Option(..., "--embed-license"),
74
+ output: Path = typer.Option(..., "--output"),
75
+ name: str | None = typer.Option(None, "--name"),
76
+ knowledge_license: str = typer.Option("internal", "--knowledge-license"),
77
+ knowledge_commercial: bool = typer.Option(False, "--knowledge-commercial"),
78
+ knowledge_redistribute: bool = typer.Option(False, "--knowledge-redistribute"),
79
+ no_attribution: bool = typer.Option(False, "--no-attribution"),
80
+ no_redistribute_embed: bool = typer.Option(False, "--no-redistribute-embed"),
81
+ embed_model: bool = typer.Option(False, "--embed-model"),
82
+ recommended_llm: str = typer.Option("", "--recommended-llm"),
83
+ sign: Path | None = typer.Option(None, "--sign", exists=True, dir_okay=False),
84
+ signer: str | None = typer.Option(None, "--signer"),
85
+ ) -> None:
86
+ """Chunk documents, embed them, and write a .tome archive."""
87
+ from tomefile.build import build_tome
88
+
89
+ embedder = _LazyEmbedder(model)
90
+ try:
91
+ build_tome(
92
+ input_dir,
93
+ output,
94
+ package_id=package_id,
95
+ revision=revision,
96
+ model=model,
97
+ embed_id=embed_id,
98
+ embed_license_name=embed_license,
99
+ embed=embedder,
100
+ name=name,
101
+ knowledge_license=knowledge_license,
102
+ knowledge_commercial=knowledge_commercial,
103
+ knowledge_redistribute=knowledge_redistribute,
104
+ attribution=not no_attribution,
105
+ redistribute_embed=not no_redistribute_embed,
106
+ embed_model=embed_model,
107
+ recommended_llm=recommended_llm,
108
+ sign_key=sign,
109
+ signer=signer,
110
+ cache_root=_cache_root(),
111
+ )
112
+ except TomeError as exc:
113
+ _fail(exc)
114
+ finally:
115
+ embedder.stop()
116
+ typer.echo(str(output))
117
+
118
+
119
+ @app.command()
120
+ def pack(
121
+ source: Path = typer.Argument(..., exists=True, dir_okay=False),
122
+ llm: str = typer.Option(..., "--llm"),
123
+ chat_license: str = typer.Option(..., "--chat-license"),
124
+ output: Path = typer.Option(..., "--output"),
125
+ chat_commercial: bool = typer.Option(False, "--chat-commercial"),
126
+ embed: bool = typer.Option(False, "--embed"),
127
+ redistributable: bool = typer.Option(False, "--redistributable"),
128
+ sign: Path | None = typer.Option(None, "--sign", exists=True, dir_okay=False),
129
+ signer: str | None = typer.Option(None, "--signer"),
130
+ ) -> None:
131
+ """Add a chat model reference to a .tome and write a .tbox."""
132
+ from tomefile.pack import pack_tome
133
+
134
+ try:
135
+ pack_tome(
136
+ source,
137
+ output,
138
+ llm=llm,
139
+ chat_license=chat_license,
140
+ cache_root=_cache_root(),
141
+ config_root=_config_root(),
142
+ chat_commercial=chat_commercial,
143
+ embed=embed,
144
+ redistributable=redistributable,
145
+ sign_key=sign,
146
+ signer=signer,
147
+ )
148
+ except TomeError as exc:
149
+ _fail(exc)
150
+ typer.echo(str(output))
151
+
152
+
153
+ @app.command()
154
+ def graph(
155
+ source: Path = typer.Argument(..., exists=True, dir_okay=False),
156
+ graph_input: Path = typer.Option(..., "--input", exists=True, dir_okay=False),
157
+ output: Path = typer.Option(..., "--output"),
158
+ ) -> None:
159
+ """Write graph tables into a .tome or .tbox."""
160
+ from tomefile.graph import install_graph
161
+
162
+ try:
163
+ install_graph(source, graph_input, output, _config_root())
164
+ except TomeError as exc:
165
+ _fail(exc)
166
+ typer.echo(str(output))
167
+
168
+
169
+ @app.command()
170
+ def inspect(
171
+ file: Path = typer.Argument(..., exists=True, dir_okay=False),
172
+ chunks: bool = typer.Option(False, "--chunks"),
173
+ ) -> None:
174
+ """Print stats for a .tome or .tbox."""
175
+ from tomefile.inspecting import inspect_archive
176
+
177
+ try:
178
+ typer.echo(inspect_archive(file, chunks=chunks))
179
+ except TomeError as exc:
180
+ _fail(exc)
181
+
182
+
183
+ @app.command()
184
+ def serve(
185
+ file: Path = typer.Argument(..., exists=True, dir_okay=False),
186
+ llm: Path | None = typer.Option(None, "--llm", exists=True, dir_okay=False),
187
+ engine: str = typer.Option("internal", "--engine"),
188
+ ollama_model: str | None = typer.Option(None, "--ollama-model"),
189
+ port: int | None = typer.Option(None, "--port"),
190
+ gpu_layers: int | None = typer.Option(None, "--gpu-layers", "-ngl"),
191
+ threads: int | None = typer.Option(None, "--threads", "-t"),
192
+ require_signature: bool = typer.Option(False, "--require-signature"),
193
+ graph: bool = typer.Option(False, "--graph"),
194
+ ) -> None:
195
+ """Serve a .tome or .tbox on an OpenAI-compatible local port."""
196
+ from tomefile.gateway import create_app
197
+
198
+ embed_server: LlamaServer | None = None
199
+ chat_server: LlamaServer | None = None
200
+ try:
201
+ prepared = prepare(
202
+ file,
203
+ cache_root=_cache_root(),
204
+ config_root=_config_root(),
205
+ require_signature=require_signature,
206
+ llm=llm,
207
+ engine=engine,
208
+ )
209
+ embed_server = LlamaServer(prepared.embed_path, embeddings=True)
210
+ embed_server.start()
211
+ if engine == "ollama":
212
+ if not ollama_model:
213
+ raise TomeError("--ollama-model is required when --engine is ollama")
214
+ check_ollama()
215
+ upstream = HttpUpstream("http://127.0.0.1:11434/v1/chat/completions", ollama_model)
216
+ else:
217
+ if ollama_model:
218
+ raise TomeError("--ollama-model is only used with --engine ollama")
219
+ assert prepared.chat_path is not None
220
+ chat_server = LlamaServer(
221
+ prepared.chat_path,
222
+ embeddings=False,
223
+ ngl=gpu_layers,
224
+ threads=threads,
225
+ )
226
+ chat_server.start()
227
+ upstream = HttpUpstream(
228
+ f"{chat_server.base_url}/v1/chat/completions",
229
+ prepared.manifest.name,
230
+ )
231
+ if graph:
232
+ from tomefile.database import has_graph
233
+
234
+ if not has_graph(prepared.db_path):
235
+ print(
236
+ "warning: --graph set but this file has no graph tables; using vectors",
237
+ file=sys.stderr,
238
+ )
239
+ gateway = create_app(
240
+ prepared.manifest,
241
+ prepared.db_path,
242
+ embed_server.embed,
243
+ upstream.complete,
244
+ upstream.stream,
245
+ use_graph=graph,
246
+ )
247
+ bind = choose_port(port)
248
+ except TomeError as exc:
249
+ if embed_server is not None:
250
+ embed_server.stop()
251
+ if chat_server is not None:
252
+ chat_server.stop()
253
+ _fail(exc)
254
+ typer.echo(f"http://127.0.0.1:{bind}/v1")
255
+ try:
256
+ uvicorn.run(gateway, host="127.0.0.1", port=bind, log_level="info")
257
+ finally:
258
+ if chat_server is not None:
259
+ chat_server.stop()
260
+ if embed_server is not None:
261
+ embed_server.stop()