tomefile 0.1.0__tar.gz

tomefile-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Simeon Emanuilov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tomefile-0.1.0/MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE README.md pyproject.toml
+graft tests
+prune tests/__pycache__

tomefile-0.1.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: tomefile
+Version: 0.1.0
+Summary: Local RAG as a single file
+Author-email: Simeon Emanuilov <simeon.emanuilov@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://tomefile.com
+Project-URL: Repository, https://github.com/s-emanuilov/tomefile
+Keywords: rag,embeddings,sqlite,local
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Environment :: Console
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typer>=0.12
+Requires-Dist: fastapi>=0.115
+Requires-Dist: uvicorn>=0.32
+Requires-Dist: httpx>=0.27
+Requires-Dist: sqlite-vec>=0.1.6
+Requires-Dist: cryptography>=43
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Dynamic: license-file
+
+# Tomefile
+
+A local RAG index you pack **into one file**.
+
+Fine-tuning a small model on your knowledgebase will not make it remember them. Retrieval will. Most RAG still calls a remote store, an embeddings API, and a set of tools. A `.tome` is the other shape: chunks, vectors, and the embedding model in one archive you open on `127.0.0.1`.
+
+Build the index once. Hand someone the file.
+
+- **`.tome`** — documents already chunked and embedded, plus the embedder (GGUF inside, or a SHA-256 reference). Checksums on every member. Optional Ed25519 signature.
+- **`.tbox`** — the same file, with a chat model packed in or pinned by hash.
+- **`tomefile serve`** — OpenAI-compatible `/v1/chat/completions` on the loopback interface. Answers cite source files and character offsets.
+
+Chunk text is plaintext SQLite; this is not a way to hide a corpus. Vector search uses the embedder named in the file. A different model with the same vector size will retrieve the wrong neighbours.
+
+Site: [tomefile.com](https://tomefile.com).

tomefile-0.1.0/README.md

@@ -0,0 +1,15 @@
+# Tomefile
+
+A local RAG index you pack **into one file**.
+
+Fine-tuning a small model on your knowledgebase will not make it remember them. Retrieval will. Most RAG still calls a remote store, an embeddings API, and a set of tools. A `.tome` is the other shape: chunks, vectors, and the embedding model in one archive you open on `127.0.0.1`.
+
+Build the index once. Hand someone the file.
+
+- **`.tome`** — documents already chunked and embedded, plus the embedder (GGUF inside, or a SHA-256 reference). Checksums on every member. Optional Ed25519 signature.
+- **`.tbox`** — the same file, with a chat model packed in or pinned by hash.
+- **`tomefile serve`** — OpenAI-compatible `/v1/chat/completions` on the loopback interface. Answers cite source files and character offsets.
+
+Chunk text is plaintext SQLite; this is not a way to hide a corpus. Vector search uses the embedder named in the file. A different model with the same vector size will retrieve the wrong neighbours.
+
+Site: [tomefile.com](https://tomefile.com).

tomefile-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[project]
+name = "tomefile"
+dynamic = ["version"]
+description = "Local RAG as a single file"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+    { name = "Simeon Emanuilov", email = "simeon.emanuilov@gmail.com" },
+]
+requires-python = ">=3.10"
+keywords = ["rag", "embeddings", "sqlite", "local"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Environment :: Console",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "typer>=0.12",
+    "fastapi>=0.115",
+    "uvicorn>=0.32",
+    "httpx>=0.27",
+    "sqlite-vec>=0.1.6",
+    "cryptography>=43",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8"]
+
+[project.urls]
+Homepage = "https://tomefile.com"
+Repository = "https://github.com/s-emanuilov/tomefile"
+
+[project.scripts]
+tomefile = "tomefile.cli:app"
+
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = { attr = "tomefile.__version__" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]

tomefile-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tomefile-0.1.0/src/tomefile/__init__.py

@@ -0,0 +1,3 @@
+"""Local RAG as a single file."""
+
+__version__ = "0.1.0"

tomefile-0.1.0/src/tomefile/archive.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import zipfile
+from pathlib import Path
+
+from tomefile.errors import TomeError
+from tomefile.hashing import sha256_bytes, sha256_file
+from tomefile.manifest import SHA_RE
+
+CHECKSUMS = "checksums.sha256"
+SIGNATURE = "signature.json"
+EXCLUDED = {CHECKSUMS, SIGNATURE}
+
+
+def checksum_document(members: dict[str, bytes]) -> bytes:
+    lines: list[str] = []
+    names = sorted((name for name in members if name not in EXCLUDED), key=lambda name: name.encode("utf-8"))
+    for name in names:
+        lines.append(f"{sha256_bytes(members[name])}  {name}")
+    return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+def parse_checksums(raw: bytes) -> dict[str, str]:
+    try:
+        text = raw.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise TomeError("checksums.sha256 is not UTF-8") from exc
+    listed: dict[str, str] = {}
+    for number, line in enumerate(text.splitlines(), start=1):
+        if line == "":
+            continue
+        digest, separator, name = line.partition("  ")  # sha256sum: digest, two spaces, name
+        if separator != "  " or SHA_RE.fullmatch(digest) is None or not _safe_name(name):
+            raise TomeError(f"checksums.sha256 line {number} is malformed")
+        if name in listed:
+            raise TomeError(f"checksums.sha256 lists {name} twice")
+        listed[name] = digest
+    return listed
+
+
+def verify_checksums(members: dict[str, bytes]) -> bytes:
+    raw = members.get(CHECKSUMS)
+    if raw is None:
+        raise TomeError("checksums.sha256 is missing")
+    listed = parse_checksums(raw)
+    payload = {name: data for name, data in members.items() if name not in EXCLUDED}
+    if set(listed) != set(payload):
+        raise TomeError("checksums.sha256 does not match the archive members")
+    for name, digest in listed.items():
+        if sha256_bytes(payload[name]) != digest:
+            raise TomeError(f"checksum mismatch for {name}")
+    return raw
+
+
+def write_archive(path: Path, members: dict[str, bytes]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
+        for name in sorted(members, key=lambda item: item.encode("utf-8")):
+            info = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0))  # stable hashes
+            info.compress_type = zipfile.ZIP_STORED
+            info.flag_bits |= 0x800  # UTF-8 names
+            info.external_attr = 0o644 << 16
+            archive.writestr(info, members[name])
+
+
+def read_archive(path: Path) -> tuple[str, dict[str, bytes]]:
+    digest = sha256_file(path)
+    try:
+        archive = zipfile.ZipFile(path)
+    except zipfile.BadZipFile as exc:
+        raise TomeError(f"{path} is not a ZIP archive") from exc
+    members: dict[str, bytes] = {}
+    with archive:
+        for info in archive.infolist():
+            if info.is_dir():
+                continue
+            if info.compress_type != zipfile.ZIP_STORED:
+                raise TomeError(f"{info.filename} is compressed; members must use store method")
+            if not _safe_name(info.filename):
+                raise TomeError(f"unsafe member {info.filename}")
+            if info.filename in members:
+                raise TomeError(f"duplicate member {info.filename}")
+            members[info.filename] = archive.read(info)
+    if not members:
+        raise TomeError(f"{path} is empty")
+    return digest, members
+
+
+def _safe_name(name: str) -> bool:
+    if name == "" or name.startswith("/") or name.endswith("/"):
+        return False
+    parts = name.split("/")
+    return ".." not in parts and "." not in parts and "" not in parts

tomefile-0.1.0/src/tomefile/build.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import tempfile
+from collections.abc import Callable
+from pathlib import Path
+
+from tomefile import __version__
+from tomefile.archive import CHECKSUMS, SIGNATURE, checksum_document, write_archive
+from tomefile.chunking import Chunk, chunk_documents, discover_documents
+from tomefile.database import create_database
+from tomefile.errors import TomeError
+from tomefile.hashing import sha256_file
+from tomefile.manifest import (
+    CHUNK_SIZE,
+    TOP_K,
+    LicenseGrant,
+    Manifest,
+    ModelRef,
+    default_name,
+    default_signer,
+    require_id,
+)
+from tomefile.models import cache_model_file, cached_model
+from tomefile.signing import sign_checksums
+
+
+def build_tome(
+    input_dir: Path,
+    output: Path,
+    *,
+    package_id: str,
+    revision: int,
+    model: Path,
+    embed_id: str,
+    embed_license_name: str,
+    embed: Callable[[str], list[float]],
+    name: str | None = None,
+    knowledge_license: str = "internal",
+    knowledge_commercial: bool = False,
+    knowledge_redistribute: bool = False,
+    attribution: bool = True,
+    redistribute_embed: bool = True,
+    embed_model: bool = False,
+    recommended_llm: str = "",
+    sign_key: Path | None = None,
+    signer: str | None = None,
+    cache_root: Path,
+) -> None:
+    package_id = require_id(package_id, "id")
+    if revision < 1:
+        raise TomeError("revision must be an integer >= 1")
+    if output.suffix.lower() != ".tome":
+        raise TomeError("output must end in .tome")
+    if not embed_id:
+        raise TomeError("--embed-id must not be empty")
+    if not embed_license_name:
+        raise TomeError("--embed-license must not be empty")
+    if embed_model and not redistribute_embed:
+        raise TomeError("--embed-model requires a redistributable embedding license")
+    documents = discover_documents(input_dir)
+    chunks = chunk_documents(documents, CHUNK_SIZE)
+    if not chunks:
+        raise TomeError("no text to index")
+    vectors = [_embed_chunk(embed, chunk) for chunk in chunks]
+    dimension = len(vectors[0])
+    digest = sha256_file(model)
+    already_cached = cached_model(cache_root, digest)
+    if not redistribute_embed and already_cached is None:
+        raise TomeError(
+            f"embedding model is not redistributable and {digest} is not in the model cache"
+        )
+    include_model = redistribute_embed and (embed_model or already_cached is None)
+    if already_cached is None and redistribute_embed:
+        cache_model_file(model, cache_root)
+    manifest = Manifest(
+        type="data_tome",
+        id=package_id,
+        revision=revision,
+        name=name or default_name(package_id),
+        embed="models/embed.gguf" if include_model else ModelRef(digest, model.name),
+        chat=None,
+        embedding_id=embed_id,
+        embedding_sha256=digest,
+        dimensions=dimension,
+        recommended_llm=recommended_llm,
+        builder="tomefile",
+        builder_version=__version__,
+        chunker="fixed-size",
+        knowledge=LicenseGrant(
+            knowledge_license or "internal",
+            knowledge_commercial,
+            knowledge_redistribute,
+            attribution,
+        ),
+        embed_license=LicenseGrant(embed_license_name, True, redistribute_embed, attribution),
+        chat_license=None,
+        chunk_size=CHUNK_SIZE,
+        retrieve_top_k=TOP_K,
+    )
+    with tempfile.TemporaryDirectory() as temporary:
+        database_path = Path(temporary) / "vectors.db"
+        create_database(
+            database_path,
+            chunks,
+            vectors,
+            {
+                "package_id": package_id,
+                "revision": str(revision),
+                "metric": "cosine",
+                "embedding_id": embed_id,
+                "embedding_sha256": digest,
+            },
+        )
+        members: dict[str, bytes] = {
+            "manifest.json": manifest.dumps(),
+            "knowledge/vectors.db": database_path.read_bytes(),
+        }
+    if include_model:
+        members["models/embed.gguf"] = model.read_bytes()
+    _finish_archive(output, members, sign_key, signer or default_signer(package_id))
+
+
+def _embed_chunk(embed: Callable[[str], list[float]], chunk: Chunk) -> list[float]:
+    try:
+        vector = embed(chunk.text)
+    except Exception as exc:
+        raise TomeError(f"embedder rejected {chunk.source}: {exc}") from exc
+    if not vector:
+        raise TomeError(f"embedder rejected {chunk.source}: empty vector")
+    return [float(value) for value in vector]
+
+
+def _finish_archive(
+    output: Path,
+    members: dict[str, bytes],
+    sign_key: Path | None,
+    signer: str,
+) -> None:
+    checksums = checksum_document(members)
+    members[CHECKSUMS] = checksums
+    if sign_key is not None:
+        members[SIGNATURE] = sign_checksums(checksums, sign_key, signer)
+    write_archive(output, members)

tomefile-0.1.0/src/tomefile/chunking.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from tomefile.errors import TomeError
+
+TEXT_SUFFIXES = {".txt", ".md"}
+
+
+@dataclass(frozen=True)
+class Chunk:
+    id: int
+    text: str
+    source: str
+    start: int
+    end: int
+
+
+def discover_documents(input_dir: Path) -> list[tuple[str, str]]:
+    found: list[tuple[str, str]] = []
+    for path in input_dir.rglob("*"):
+        if not path.is_file() or path.suffix.lower() not in TEXT_SUFFIXES:
+            continue
+        relative = path.relative_to(input_dir).as_posix()
+        try:
+            text = path.read_text(encoding="utf-8")
+        except UnicodeDecodeError as exc:
+            raise TomeError(f"{relative} is not valid UTF-8") from exc
+        found.append((relative, text))
+    found.sort(key=lambda item: item[0].encode("utf-8"))
+    return found
+
+
+def chunk_text(text: str, size: int) -> list[tuple[int, int, str]]:
+    if size < 1:
+        raise TomeError("chunk size must be positive")
+    pieces: list[tuple[int, int, str]] = []
+    index = 0
+    length = len(text)
+    while index < length:
+        end = min(index + size, length)
+        if end < length:
+            # last whitespace in the window; keep it in this chunk
+            window = text[index:end]
+            cut = -1
+            for offset, char in enumerate(window):
+                if char.isspace():
+                    cut = offset
+            if cut >= 0:
+                end = index + cut + 1
+        if end <= index:
+            end = index + 1
+        pieces.append((index, end, text[index:end]))
+        index = end
+    return pieces
+
+
+def chunk_documents(documents: list[tuple[str, str]], size: int) -> list[Chunk]:
+    chunks: list[Chunk] = []
+    next_id = 1
+    for source, text in documents:
+        for start, end, piece in chunk_text(text, size):
+            chunks.append(Chunk(next_id, piece, source, start, end))
+            next_id += 1
+    return chunks