lnclite 0.1.0__tar.gz

lnclite-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 AllenChou
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

lnclite-0.1.0/PKG-INFO

@@ -0,0 +1,101 @@
+Metadata-Version: 2.4
+Name: lnclite
+Version: 0.1.0
+Summary: Lite usages of lancedb.
+License-Expression: MIT
+License-File: LICENSE
+Author: Allen Chou
+Author-email: f1470891079@gmail.com
+Requires-Python: >=3.11,<4
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: lancedb
+Requires-Dist: openai
+Requires-Dist: openai-embeddings-model
+Requires-Dist: paginatic
+Requires-Dist: pydantic (>=2)
+Requires-Dist: xxhash
+Project-URL: Homepage, https://github.com/allen2c/lnclite
+Project-URL: PyPI, https://pypi.org/project/lnclite/
+Project-URL: Repository, https://github.com/allen2c/lnclite
+Description-Content-Type: text/markdown
+
+# lnclite
+
+`lnclite` is a small async LanceDB document store for OpenAI-compatible embeddings. It gives you a compact API for creating a local vector database, adding documents, filtering by tags, and running semantic search.
+
+## Installation
+
+```bash
+pip install lnclite
+```
+
+For local development from this repository:
+
+```bash
+poetry install --all-groups
+```
+
+## Quick Start
+
+```python
+import asyncio
+
+from openai import AsyncOpenAI
+from openai_embeddings_model import ModelSettings
+
+from lnclite import DocumentCreate, Lnclite, get_openai_embeddings_model
+
+
+async def main():
+    embeddings = get_openai_embeddings_model(
+        openai_client=AsyncOpenAI(),
+    )
+
+    client = await Lnclite.new(
+        lancedb_path="outputs/demo.lance",
+        openai_embeddings_model=embeddings,
+        model_settings=ModelSettings(dimensions=1536),
+    )
+
+    await client.documents.batch_create(
+        [
+            DocumentCreate(
+                content="A note about async Python clients.",
+                tags=["type:note", "topic:python"],
+            ),
+            DocumentCreate(
+                content="A note about vector search and indexing.",
+                tags=["type:note", "topic:search"],
+            ),
+        ]
+    )
+
+    await client.create_index()
+
+    results = await client.search(
+        "How should I design vector search?",
+        tags_any=["topic:search"],
+    )
+
+    for result in results.results:
+        print(result.document.content)
+        print(result.document.tags)
+        print(result.distance)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Documentation
+
+Full documentation is published with MkDocs Material from this repository's `docs/` directory.
+
+## License
+
+MIT
+

lnclite-0.1.0/README.md

@@ -0,0 +1,75 @@
+# lnclite
+
+`lnclite` is a small async LanceDB document store for OpenAI-compatible embeddings. It gives you a compact API for creating a local vector database, adding documents, filtering by tags, and running semantic search.
+
+## Installation
+
+```bash
+pip install lnclite
+```
+
+For local development from this repository:
+
+```bash
+poetry install --all-groups
+```
+
+## Quick Start
+
+```python
+import asyncio
+
+from openai import AsyncOpenAI
+from openai_embeddings_model import ModelSettings
+
+from lnclite import DocumentCreate, Lnclite, get_openai_embeddings_model
+
+
+async def main():
+    embeddings = get_openai_embeddings_model(
+        openai_client=AsyncOpenAI(),
+    )
+
+    client = await Lnclite.new(
+        lancedb_path="outputs/demo.lance",
+        openai_embeddings_model=embeddings,
+        model_settings=ModelSettings(dimensions=1536),
+    )
+
+    await client.documents.batch_create(
+        [
+            DocumentCreate(
+                content="A note about async Python clients.",
+                tags=["type:note", "topic:python"],
+            ),
+            DocumentCreate(
+                content="A note about vector search and indexing.",
+                tags=["type:note", "topic:search"],
+            ),
+        ]
+    )
+
+    await client.create_index()
+
+    results = await client.search(
+        "How should I design vector search?",
+        tags_any=["topic:search"],
+    )
+
+    for result in results.results:
+        print(result.document.content)
+        print(result.document.tags)
+        print(result.distance)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Documentation
+
+Full documentation is published with MkDocs Material from this repository's `docs/` directory.
+
+## License
+
+MIT

lnclite-0.1.0/lnclite/__init__.py

@@ -0,0 +1,826 @@
+"""Small async LanceDB document store with OpenAI embeddings."""
+
+import functools
+import hashlib
+import logging
+import time
+from pathlib import Path
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Final,
+    List,
+    Literal,
+    Optional,
+    Text,
+    Type,
+)
+
+import diskcache
+import lancedb
+import lancedb.index
+import numpy as np
+import tiktoken
+from lancedb.pydantic import LanceModel, Vector
+from openai import AsyncOpenAI
+from openai_embeddings_model import MAX_BATCH_SIZE as DEFAULT_EMBEDDINGS_MAX_BATCH_SIZE
+from openai_embeddings_model import (
+    AsyncOpenAIEmbeddingsModel,
+    ModelSettings,
+)
+from openai_embeddings_model.normalize import normalize
+from paginatic import TokenPaginatic
+from paginatic.helpers import decode_and_verify, encode_and_sign
+from pydantic import BaseModel, Field, model_validator
+
+if TYPE_CHECKING:
+    from lnclite.file_ingestor import FileReader
+
+__version__: Final[Text] = "0.1.0"
+__all__: Final[List[Text]] = [
+    "Document",
+    "DocumentCreate",
+    "Lnclite",
+    "LncliteNotFoundError",
+    "ManifestModel",
+    "SearchResult",
+    "SearchResults",
+    "get_model_settings",
+    "get_openai_embeddings_model",
+]
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MANIFEST_TABLE = "manifest"
+DEFAULT_DOCUMENT_TABLE = "documents"
+DEFAULT_OPENAI_MODEL = "text-embedding-3-small"
+DEFAULT_MAX_INPUT_TOKENS = 4096
+DEFAULT_DIMENSIONS = 1536
+
+VectorIndexPreference = Literal["storage", "balanced", "accuracy", "latency"]
+ListOrder = Literal["asc", "desc", 1, -1]
+SqlOrder = Literal["ASC", "DESC"]
+
+
+def gen_id() -> str:
+    from lnclite.utils.snowflake import generate_id
+
+    return generate_id()
+
+
+@functools.cache
+def get_document_lancedb_model(dim: int) -> Type[LanceModel]:
+    class DocumentLancedbModel(LanceModel):
+        id: int = Field(default_factory=gen_id)
+        content: Text = Field(description="The content of the document.")  # noqa: E501
+        md5: Text = ""
+        vector: Vector(dim)
+        tags: List[Text] = Field(default_factory=list)
+
+        @model_validator(mode="after")
+        def validate_values(self) -> "DocumentLancedbModel":
+            self.content = self.content.strip()
+            if not self.content:
+                raise ValueError("Content cannot be empty")
+            self.md5 = hashlib.md5(self.content.encode()).hexdigest()
+            return self
+
+    return DocumentLancedbModel
+
+
+@functools.cache
+def get_openai_client() -> AsyncOpenAI:
+    return AsyncOpenAI()
+
+
+@functools.cache
+def get_embeddings_cache() -> diskcache.Cache:
+    return diskcache.Cache(".cache/embeddings")
+
+
+@functools.cache
+def get_encoding_for_model(model: str) -> tiktoken.Encoding:
+    try:
+        return tiktoken.encoding_for_model(model)
+    except KeyError:
+        logger.warning(
+            f"Encoding for model {model} not found, using default encoding 'gpt-5"
+        )
+        return tiktoken.encoding_for_model("gpt-5")
+
+
+def get_openai_embeddings_model(
+    model: str = DEFAULT_OPENAI_MODEL,
+    openai_client: AsyncOpenAI | None = None,
+    cache: diskcache.Cache | None = None,
+    encoding: tiktoken.Encoding | None = None,
+    max_batch_size: int = DEFAULT_EMBEDDINGS_MAX_BATCH_SIZE,
+    max_input_tokens: int = DEFAULT_MAX_INPUT_TOKENS,
+) -> "AsyncOpenAIEmbeddingsModel":
+    return AsyncOpenAIEmbeddingsModel(
+        model=model,
+        openai_client=openai_client or get_openai_client(),
+        cache=cache or get_embeddings_cache(),
+        encoding=encoding or get_encoding_for_model(model),
+        max_batch_size=max_batch_size,
+        max_input_tokens=max_input_tokens,
+    )
+
+
+def get_model_settings(dimensions: int = DEFAULT_DIMENSIONS) -> "ModelSettings":
+    return ModelSettings(dimensions=dimensions)
+
+
+def quote_sql_string(s: str) -> str:
+    return "'" + s.replace("'", "''") + "'"
+
+
+def tag_filter_any(tags: list[str]) -> str:
+    values = ", ".join(quote_sql_string(tag) for tag in tags)
+    return f"array_has_any(tags, [{values}])"
+
+
+def tag_filter_all(tags: list[str]) -> str:
+    values = ", ".join(quote_sql_string(tag) for tag in tags)
+    return f"array_has_all(tags, [{values}])"
+
+
+def recommended_vector_index_config(
+    row_count: int,
+    dim: int,
+    *,
+    prefer: VectorIndexPreference = "balanced",
+):
+    """Return a LanceDB vector index config for dot-search.
+
+    Assumes document vectors and query vectors are normalized.
+    Returns None when brute-force is better or when there are not enough rows.
+    """
+
+    # Too small. Brute-force is exact and fast.
+    # Also avoids PQ training errors.
+    if row_count < 256:
+        return None
+
+    # Still small. Brute-force is usually fine.
+    # If you really want an index, IvfFlat is safer than PQ.
+    if row_count < 10_000:
+        if prefer in {"accuracy", "latency"}:
+            return lancedb.index.IvfFlat(
+                distance_type="dot",
+                num_partitions=32,
+            )
+        return None
+
+    if row_count < 50_000:
+        return lancedb.index.IvfFlat(
+            distance_type="dot",
+            num_partitions=128,
+        )
+
+    if row_count < 100_000:
+        if prefer == "storage":
+            return lancedb.index.IvfPq(
+                distance_type="dot",
+                num_partitions=256,
+                num_sub_vectors=recommended_num_sub_vectors(dim, prefer),
+                num_bits=8,
+            )
+
+        return lancedb.index.IvfFlat(
+            distance_type="dot",
+            num_partitions=256,
+        )
+
+    if row_count < 500_000:
+        return lancedb.index.IvfPq(
+            distance_type="dot",
+            num_partitions=1024 if prefer == "storage" else 2048,
+            num_sub_vectors=recommended_num_sub_vectors(dim, prefer),
+            num_bits=8,
+        )
+
+    if row_count < 1_000_000:
+        return lancedb.index.IvfPq(
+            distance_type="dot",
+            num_partitions=2048 if prefer == "storage" else 4096,
+            num_sub_vectors=recommended_num_sub_vectors(dim, prefer),
+            num_bits=8,
+        )
+
+    if prefer == "latency":
+        return lancedb.index.HnswPq(
+            distance_type="dot",
+            m=20,
+            ef_construction=300,
+            num_sub_vectors=recommended_num_sub_vectors(dim, "balanced"),
+            num_bits=8,
+        )
+
+    return lancedb.index.IvfPq(
+        distance_type="dot",
+        num_partitions=4096 if prefer in {"storage", "balanced"} else 8192,
+        num_sub_vectors=recommended_num_sub_vectors(dim, prefer),
+        num_bits=8,
+    )
+
+
+def recommended_num_sub_vectors(
+    dim: int, prefer: VectorIndexPreference = "balanced"
+) -> int:
+    """Return a PQ subvector count.
+
+    Higher = more accurate, larger index.
+    Lower = more compressed, lower recall.
+    """
+
+    # Prefer subvector sizes around 8~16 dimensions.
+    if prefer == "storage":
+        target_sub_dim = 16
+    elif prefer == "accuracy":
+        target_sub_dim = 8
+    else:
+        target_sub_dim = 12
+
+    candidates = [x for x in range(1, dim + 1) if dim % x == 0]
+    return min(candidates, key=lambda x: abs((dim / x) - target_sub_dim))
+
+
+async def get_default_dimensions(
+    openai_embeddings_model: AsyncOpenAIEmbeddingsModel,
+) -> int:
+    emb_result = await openai_embeddings_model.get_embeddings(
+        input="Hello, world!",
+        model_settings=ModelSettings(),
+    )
+    return emb_result.to_numpy().shape[1]
+
+
+class ManifestLancedbModel(LanceModel):
+    id: int = Field(default_factory=gen_id)
+    name: Text = Field(description="The name of the database.")
+    description: Text = Field(description="The description of the database.")
+    model: Text = Field(description="The embedding model name.")
+    dimensions: int = Field(description="The dimensions of the embeddings.")
+    last_updated: int = Field(default_factory=lambda: int(time.time()))
+
+
+class ManifestModel(LanceModel):
+    id: int
+    name: Text
+    description: Text
+    model: Text
+    dimensions: int
+    last_updated: int
+
+
+class DocumentCreate(BaseModel):
+    content: Text
+    tags: List[Text] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def validate_values(self) -> "DocumentCreate":
+        self.content = self.content.strip()
+        if not self.content:
+            raise ValueError("Content cannot be empty")
+        return self
+
+
+class Document(BaseModel):
+    id: int
+    content: Text
+    md5: Text
+    vector: Optional[List[float]]
+    tags: List[Text]
+
+
+class Lnclite:
+    def __init__(
+        self,
+        lancedb_path: Path | str | None = None,
+        *,
+        manifest_table: Text = DEFAULT_MANIFEST_TABLE,
+        document_table: Text = DEFAULT_DOCUMENT_TABLE,
+        openai_embeddings_model: "AsyncOpenAIEmbeddingsModel",
+        model_settings: "ModelSettings",
+        token_secret_key: Text = "__lnclite__",
+        vector_search_prefer: VectorIndexPreference = "balanced",
+        verbose: bool = False,
+    ):
+        self.lancedb_path = Path(lancedb_path)
+        self._connection: lancedb.AsyncConnection | None = None
+
+        self.manifest_table = manifest_table
+        self.document_table = document_table
+
+        self.openai_embeddings_model = openai_embeddings_model
+        self.model_settings = model_settings
+        self.max_tokens = self.openai_embeddings_model._max_input_tokens
+
+        self._secret_key = token_secret_key
+        self.vector_search_prefer = vector_search_prefer
+        self.verbose = verbose
+
+        if self.model_settings.dimensions is None:
+            raise ValueError("Model settings dimensions is not set")
+        self._document_lancedb_model: Type[LanceModel] = get_document_lancedb_model(
+            self.model_settings.dimensions
+        )
+        self._manifest_lancedb_model = ManifestLancedbModel
+
+    @classmethod
+    async def new(
+        cls,
+        lancedb_path: Path | str,
+        *,
+        manifest_table: Text = DEFAULT_MANIFEST_TABLE,
+        document_table: Text = DEFAULT_DOCUMENT_TABLE,
+        openai_embeddings_model: "AsyncOpenAIEmbeddingsModel",
+        model_settings: "ModelSettings",
+        token_secret_key: Text = "__lnclite__",
+        vector_search_prefer: VectorIndexPreference = "balanced",
+        verbose: bool = False,
+    ) -> "Lnclite":
+        lancedb_path = Path(lancedb_path)
+        if lancedb_path.is_dir():
+            # If not a empty directory, raise an error
+            for _ in lancedb_path.iterdir():
+                raise ValueError(f"Lancedb path {lancedb_path} already exists ")
+        else:
+            lancedb_path.mkdir(parents=True, exist_ok=True)
+
+        if model_settings.dimensions is None:
+            model_settings.dimensions = await get_default_dimensions(
+                openai_embeddings_model
+            )
+
+        return cls(
+            lancedb_path=lancedb_path,
+            manifest_table=manifest_table,
+            document_table=document_table,
+            openai_embeddings_model=openai_embeddings_model,
+            model_settings=model_settings,
+            token_secret_key=token_secret_key,
+            vector_search_prefer=vector_search_prefer,
+            verbose=verbose,
+        )
+
+    @classmethod
+    async def new_from_dir(
+        cls,
+        dir_path: Path | str,
+        lancedb_path: Path | str,
+        *,
+        dataset_name: Text,
+        dataset_description: Text,
+        manifest_table: Text = DEFAULT_MANIFEST_TABLE,
+        document_table: Text = DEFAULT_DOCUMENT_TABLE,
+        openai_embeddings_model: "AsyncOpenAIEmbeddingsModel",
+        model_settings: "ModelSettings",
+        extension_readers: Optional[Dict[str, "FileReader"]] = None,
+        batch_size: int = 100,
+    ) -> "Lnclite":
+        from lnclite.file_ingestor import FileIngestor
+
+        dir_path = Path(dir_path)
+        if not dir_path.is_dir():
+            raise FileNotFoundError(f"Directory {dir_path} not found")
+
+        lancedb_path = Path(lancedb_path)
+        if lancedb_path.is_dir():
+            # If not a empty directory, raise an error
+            for _ in lancedb_path.iterdir():
+                raise ValueError(f"Lancedb path {lancedb_path} already exists ")
+
+        if model_settings.dimensions is None:
+            model_settings.dimensions = await get_default_dimensions(
+                openai_embeddings_model
+            )
+
+        lnclite = cls(
+            lancedb_path=lancedb_path,
+            manifest_table=manifest_table,
+            document_table=document_table,
+            openai_embeddings_model=openai_embeddings_model,
+            model_settings=model_settings,
+        )
+
+        # Create manifest
+        await lnclite.manifest.upsert(
+            name=dataset_name,
+            description=dataset_description,
+            model=openai_embeddings_model.model,
+            dimensions=model_settings.dimensions,
+        )
+
+        # Create documents
+        file_ingestor = FileIngestor()
+        if extension_readers is not None:
+            for extension, reader in extension_readers.items():
+                file_ingestor.register_reader(extension, reader)
+
+        batch: List[DocumentCreate] = []
+        async for file in file_ingestor.ingest_async(dir_path):
+            _file_content = file["content"].strip()
+            _file_path = str(file["path"])
+
+            if not _file_content:
+                logger.warning(f"Skipping {_file_path} due to empty content")
+                continue
+
+            relative_path = Path(_file_path).relative_to(dir_path).as_posix()
+            batch.append(
+                DocumentCreate(
+                    content=_file_content,
+                    tags=[f"path:{relative_path}"],
+                )
+            )
+
+            if len(batch) >= batch_size:
+                await lnclite.documents.batch_create(batch)
+                logger.info(f"Created {len(batch)} documents")
+                batch = []
+
+        if batch:
+            await lnclite.documents.batch_create(batch)
+            logger.info(f"Created {len(batch)} documents")
+
+        await lnclite.documents.create_index()
+
+        return lnclite
+
+    @classmethod
+    async def load(
+        cls,
+        lancedb_path: Path | str,
+        *,
+        manifest_table: Text = DEFAULT_MANIFEST_TABLE,
+        document_table: Text = DEFAULT_DOCUMENT_TABLE,
+        openai_embeddings_model: "AsyncOpenAIEmbeddingsModel",
+        model_settings: "ModelSettings",
+        vector_search_prefer: VectorIndexPreference = "balanced",
+        refresh_index: bool = False,
+        verbose: bool = False,
+    ) -> "Lnclite":
+        lancedb_path = Path(lancedb_path)
+        if not lancedb_path.is_dir():
+            raise FileNotFoundError(f"Lancedb path {lancedb_path} not found")
+
+        if model_settings.dimensions is None:
+            model_settings.dimensions = await get_default_dimensions(
+                openai_embeddings_model
+            )
+
+        lnclite = cls(
+            lancedb_path=lancedb_path,
+            manifest_table=manifest_table,
+            document_table=document_table,
+            openai_embeddings_model=openai_embeddings_model,
+            model_settings=model_settings,
+            vector_search_prefer=vector_search_prefer,
+            verbose=verbose,
+        )
+
+        # Validate manifest
+        manifest = await lnclite.manifest.get()
+        if manifest is None:
+            raise LncliteNotFoundError("Manifest not found")
+        if manifest.model != openai_embeddings_model.model:
+            raise ValueError(
+                f"OpenAI embeddings model mismatch: {manifest.model} != {openai_embeddings_model.model}"  # noqa: E501
+            )
+        if manifest.dimensions != model_settings.dimensions:
+            raise ValueError(
+                f"Model settings dimensions mismatch: {manifest.dimensions} != {model_settings.dimensions}"  # noqa: E501
+            )
+
+        if refresh_index:
+            await lnclite.documents.create_index()
+
+        return lnclite
+
+    async def get_connection(self) -> lancedb.AsyncConnection:
+        if self._connection is None:
+            self._connection = await lancedb.connect_async(self.lancedb_path)
+            logger.info(f"Lancedb connected to {self.lancedb_path}")
+        return self._connection
+
+    @functools.cached_property
+    def manifest(self) -> "Manifest":
+        return Manifest(self)
+
+    @functools.cached_property
+    def documents(self) -> "Documents":
+        return Documents(self)
+
+    async def create_index(self) -> None:
+        await self.documents.create_index()
+
+    async def embed(self, texts: List[Text]) -> np.ndarray:
+        emb_res = await self.openai_embeddings_model.get_embeddings(
+            texts, model_settings=self.model_settings
+        )
+        return normalize(emb_res.to_numpy())  # (n, d)
+
+    async def search(
+        self,
+        query: Text,
+        *,
+        tags_any: Optional[List[Text]] = None,
+        tags_all: Optional[List[Text]] = None,
+        limit: int = 5,
+        verbose: bool = False,
+    ) -> "SearchResults":
+        document_table = await self.documents.get_table()
+
+        query_vector = (await self.embed([query]))[0]
+
+        search_query = await document_table.search(query_vector)
+        tags_filter = _tags_filter(tags_any=tags_any, tags_all=tags_all)
+        if tags_filter is not None:
+            search_query = search_query.where(tags_filter)
+
+        if verbose or self.verbose:
+            logger.info(f"Query plan: {await search_query.explain_plan()}")
+
+        search_results: List[Dict] = (
+            await search_query.distance_type("dot").limit(limit).to_list()
+        )
+
+        results: List[SearchResult] = []
+        for result in search_results:
+            _doc = Document.model_validate(result)
+            _distance = result["_distance"]
+            results.append(SearchResult(document=_doc, distance=_distance))
+
+        return SearchResults(results=results)
+
+
+class Manifest:
+    def __init__(self, client: "Lnclite"):
+        self.client = client
+        self._table: lancedb.AsyncTable | None = None
+
+    async def get_table(self) -> lancedb.AsyncTable:
+        if self._table is not None:
+            return self._table
+
+        conn = await self.client.get_connection()
+        if await _table_exists(conn, self.client.manifest_table):
+            self._table = await conn.open_table(self.client.manifest_table)
+        else:
+            self._table = await conn.create_table(
+                self.client.manifest_table, schema=self.client._manifest_lancedb_model
+            )
+        return self._table
+
+    async def get(self) -> ManifestModel | None:
+        manifest_table = await self.get_table()
+        _query_builder = manifest_table.query()
+        manifests = await _query_builder.limit(1).to_pydantic(
+            self.client._manifest_lancedb_model
+        )
+        if len(manifests) > 0:
+            return ManifestModel.model_validate_json(manifests[0].model_dump_json())
+        return None
+
+    async def retrieve(self) -> ManifestModel:
+        might_manifest = await self.get()
+        if might_manifest is not None:
+            return might_manifest
+        raise LncliteNotFoundError("Manifest not found")
+
+    async def upsert(
+        self,
+        *,
+        name: Text,
+        description: Text,
+        model: Text,
+        dimensions: int,
+    ) -> ManifestModel:
+        table = await self.get_table()
+        might_manifest = await self.get()
+
+        if might_manifest is None:
+            manifest = self.client._manifest_lancedb_model(
+                name=name, description=description, model=model, dimensions=dimensions
+            )
+            await table.add([manifest])
+
+        else:
+            manifest = might_manifest
+            manifest.name = name
+            manifest.description = description
+            manifest.model = model
+            manifest.dimensions = dimensions
+            manifest.last_updated = int(time.time())
+            await table.update(where=f"id = {manifest.id}", values=manifest)
+
+        return ManifestModel.model_validate_json(manifest.model_dump_json())
+
+
+class Documents:
+    def __init__(self, client: "Lnclite"):
+        self.client = client
+        self._table: lancedb.AsyncTable | None = None
+
+    async def get_table(self) -> lancedb.AsyncTable:
+        if self._table is not None:
+            return self._table
+
+        conn = await self.client.get_connection()
+        if await _table_exists(conn, self.client.document_table):
+            self._table = await conn.open_table(self.client.document_table)
+        else:
+            self._table = await conn.create_table(
+                self.client.document_table, schema=self.client._document_lancedb_model
+            )
+        return self._table
+
+    async def create_index(self) -> None:
+        table = await self.get_table()
+
+        await table.create_index("tags", config=lancedb.index.LabelList())
+
+        row_count = await self.count()
+
+        vs_config = recommended_vector_index_config(
+            row_count,
+            self.client.model_settings.dimensions,
+            prefer=self.client.vector_search_prefer,
+        )
+        if vs_config is None:
+            logger.info(
+                "Skipping vector index: row_count=%s is too small; brute-force search is exact and fast",  # noqa: E501
+                row_count,
+            )
+        else:
+            await table.create_index("vector", config=vs_config)
+            logger.info(f"Created vector index with config: {vs_config}")
+
+    async def count(self) -> int:
+        document_table = await self.get_table()
+        return await document_table.count_rows()
+
+    async def get(self, id: int) -> Document | None:
+        document_table = await self.get_table()
+        documents = await (
+            document_table.query()
+            .where(f"id = {id}")
+            .limit(1)
+            .to_pydantic(self.client._document_lancedb_model)
+        )
+        if len(documents) > 0:
+            return Document.model_validate_json(documents[0].model_dump_json())
+        return None
+
+    async def retrieve(self, id: int) -> Document:
+        might_document = await self.get(id)
+        if might_document is not None:
+            return might_document
+        raise LncliteNotFoundError(f"Document with id {id} not found")
+
+    async def create(self, document_create: DocumentCreate) -> Document:
+        return (await self.batch_create([document_create]))[0]
+
+    async def batch_create(
+        self, document_creates: List[DocumentCreate]
+    ) -> List[Document]:
+        document_table = await self.get_table()
+
+        normalized_vectors = await self.client.embed(
+            [d.content for d in document_creates]
+        )
+
+        documents = [
+            self.client._document_lancedb_model(
+                content=d.content, tags=d.tags, vector=v
+            )
+            for d, v in zip(document_creates, normalized_vectors)
+        ]
+
+        await document_table.add(documents)
+
+        output: List[Document] = []
+        for document, v in zip(documents, normalized_vectors):
+            _doc = Document.model_validate_json(
+                document.model_dump_json(exclude_none=True)
+            )
+            _doc.vector = v.tolist()
+            output.append(_doc)
+
+        return output
+
+    async def list(
+        self,
+        *,
+        tags_any: Optional[List[Text]] = None,
+        tags_all: Optional[List[Text]] = None,
+        limit: int = 10,
+        order: ListOrder = "asc",
+        next_page_token: Optional[Text] = None,
+        verbose: bool = False,
+    ) -> TokenPaginatic[Document]:
+        if limit < 1:
+            raise ValueError(f"Limit must be greater than 0, got {limit}")
+
+        sql_order = _to_sql_order(order)
+        id_operator = ">" if sql_order == "ASC" else "<"
+        after_id: Optional[int] = None
+        if next_page_token is not None:
+            decoded_token = decode_and_verify(next_page_token, self.client._secret_key)
+            after_id = decoded_token.get("after")
+
+        document_table = await self.client.documents.get_table()
+
+        # Prepare query
+        query_builder = document_table.query()
+
+        query_builder = query_builder.where(
+            _documents_list_where_clause(
+                id_operator=id_operator,
+                sql_order=sql_order,
+                after_id=after_id,
+                tags_any=tags_any,
+                tags_all=tags_all,
+            )
+        ).limit(limit + 1)
+        if verbose or self.client.verbose:
+            logger.info(f"Query plan: {await query_builder.explain_plan()}")
+
+        # Execute query
+        documents = await query_builder.to_pydantic(self.client._document_lancedb_model)
+
+        has_more = len(documents) > limit
+        documents = documents[:limit]
+
+        _next_token = (
+            encode_and_sign({"after": documents[-1].id}, self.client._secret_key)
+            if has_more
+            else None
+        )
+
+        return TokenPaginatic(
+            object="list",
+            data=[Document.model_validate_json(d.model_dump_json()) for d in documents],
+            next_page_token=_next_token,
+        )
+
+
+class SearchResult(BaseModel):
+    document: Document
+    distance: float
+
+
+class SearchResults(BaseModel):
+    results: List[SearchResult]
+
+
+class LncliteNotFoundError(Exception):
+    pass
+
+
+def _documents_list_where_clause(
+    *,
+    id_operator: Literal[">", "<"],
+    sql_order: SqlOrder,
+    after_id: Optional[int] = None,
+    tags_any: Optional[List[Text]] = None,
+    tags_all: Optional[List[Text]] = None,
+) -> str:
+    id_filter = f"id {id_operator} {after_id}" if after_id is not None else "id > 0"
+    filters = [id_filter]
+
+    if tags_filter := _tags_filter(tags_any=tags_any, tags_all=tags_all):
+        filters.append(tags_filter)
+
+    where_clause = " AND ".join(f"({filter_})" for filter_ in filters)
+    return f"{where_clause} ORDER BY id {sql_order}"
+
+
+def _tags_filter(
+    *,
+    tags_any: Optional[List[Text]] = None,
+    tags_all: Optional[List[Text]] = None,
+) -> Optional[str]:
+    filters: List[str] = []
+    if tags_any:
+        filters.append(tag_filter_any(tags_any))
+    if tags_all:
+        filters.append(tag_filter_all(tags_all))
+    if not filters:
+        return None
+    return " AND ".join(f"({filter_})" for filter_ in filters)
+
+
+def _to_sql_order(order: ListOrder) -> SqlOrder:
+    if order in ("asc", 1):
+        return "ASC"
+    if order in ("desc", -1):
+        return "DESC"
+    raise ValueError(f"Invalid order: {order}")
+
+
+async def _table_exists(conn: lancedb.AsyncConnection, table_name: Text) -> bool:
+    table_list = await conn.list_tables()
+    return table_name in table_list.tables

lnclite-0.1.0/lnclite/file_ingestor.py

@@ -0,0 +1,133 @@
+"""Directory crawling and text extraction from readable files.
+
+Provides FilesIngestor for scanning trees while skipping binary and hidden paths.
+"""
+
+import asyncio
+import logging
+from pathlib import Path
+from typing import (
+    AsyncGenerator,
+    Awaitable,
+    Callable,
+    Dict,
+    Generator,
+    TypeAlias,
+    TypedDict,
+    cast,
+)
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+DEFAULT_BINARY_PROBE_CHUNK_SIZE: int = 1024
+
+FileReader: TypeAlias = Callable[[Path], str] | Callable[[Path], Awaitable[str]]
+
+
+class FileIngestorResult(TypedDict):
+    path: str
+    content: str
+
+
+class FileIngestor:
+    """
+    A utility class to crawl directories and extract text content
+    from readable files while filtering out binary and hidden data.
+    """
+
+    def __init__(self) -> None:
+        # Maps file extensions to specific parsing functions
+        self._custom_readers: Dict[str, FileReader] = {}
+
+    def register_reader(self, extension: str, reader_func: FileReader) -> None:
+        """Registers a handler for a specific file extension (e.g., '.pdf')."""
+        self._custom_readers[extension.lower()] = reader_func
+
+    def ingest(self, dir_path: str) -> Generator[FileIngestorResult, None, None]:
+        """
+        Iterates through the directory and yields documents as they are processed.
+        Yields:
+            Dict containing 'path' and 'content' keys.
+        """
+        root = Path(dir_path)
+
+        for file_path in root.rglob("*"):
+            # Ensure it is a file and not an excluded path
+            if not file_path.is_file() or self._is_excluded(file_path):
+                continue
+
+            extension = file_path.suffix.lower()
+
+            try:
+                # 1. Use specialized reader if registered
+                if extension in self._custom_readers:
+                    reader = self._custom_readers[extension]
+                    if asyncio.iscoroutinefunction(reader):
+                        logger.warning(
+                            "Skipping %s: async reader requires ingest_async.",
+                            file_path,
+                        )
+                        continue
+                    sync_reader = cast(Callable[[Path], str], reader)
+                    content = sync_reader(file_path)
+                    yield FileIngestorResult(path=str(file_path), content=content)
+
+                # 2. Fallback to binary probe for generic text files
+                elif not self._is_binary(file_path):
+                    content = self._read_text(file_path)
+                    yield FileIngestorResult(path=str(file_path), content=content)
+
+            except Exception as e:
+                logger.warning("Skipping %s due to error: %s", file_path, e)
+
+    async def ingest_async(
+        self, dir_path: str
+    ) -> AsyncGenerator[FileIngestorResult, None]:
+        """Async variant of ingest: walks the tree sync, reads in worker threads."""
+        root = Path(dir_path)
+
+        for file_path in root.rglob("*"):
+            if not file_path.is_file() or self._is_excluded(file_path):
+                continue
+
+            extension = file_path.suffix.lower()
+
+            try:
+                if extension in self._custom_readers:
+                    reader = self._custom_readers[extension]
+                    is_coro_reader = asyncio.iscoroutinefunction(reader)
+                    if is_coro_reader:
+                        content = await reader(file_path)
+                    else:
+                        content = reader(file_path)
+                    yield FileIngestorResult(path=str(file_path), content=content)
+                elif not await asyncio.to_thread(self._is_binary, file_path):
+                    content = await asyncio.to_thread(self._read_text, file_path)
+                    yield FileIngestorResult(path=str(file_path), content=content)
+            except Exception as e:
+                logger.warning("Skipping %s due to error: %s", file_path, e)
+
+    def _is_binary(
+        self, file_path: Path, chunk_size: int = DEFAULT_BINARY_PROBE_CHUNK_SIZE
+    ) -> bool:
+        """Determines if a file is binary using null-byte detection and UTF-8 probing."""  # noqa: E501
+        try:
+            with open(file_path, "rb") as f:
+                chunk = f.read(chunk_size)
+                # Null bytes are standard in binary formats
+                if b"\0" in chunk:
+                    return True
+                # Attempt decoding to verify it's a valid text format
+                chunk.decode("utf-8")
+                return False
+        except (UnicodeDecodeError, Exception):
+            return True
+
+    def _is_excluded(self, path: Path) -> bool:
+        """Checks if any part of the file path starts with '.' or '_'."""
+        return any(part.startswith((".", "_")) for part in path.parts)
+
+    def _read_text(self, file_path: Path) -> str:
+        """Reads plain text files with encoding safety."""
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+            return f.read()

lnclite-0.1.0/lnclite/utils/calculate_index_params.py

@@ -0,0 +1,18 @@
+"""Index parameter helpers for LanceDB vector indexes."""
+
+import math
+
+
+def calculate_index_params(n_rows: int, dimension: int) -> tuple[int, int]:
+    max_partitions = n_rows // 256
+    num_partitions = min(int(math.sqrt(n_rows) * 8), max_partitions)
+    num_partitions = max(num_partitions, 1)
+
+    target = dimension // 8
+    num_sub_vectors = 1
+    for i in range(target, 0, -1):
+        if dimension % i == 0:
+            num_sub_vectors = i
+            break
+
+    return num_partitions, num_sub_vectors

lnclite-0.1.0/lnclite/utils/get_file_fp.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+import xxhash
+
+
+def get_file_fp(file_path: Path | str) -> str:
+    hasher = xxhash.xxh64()
+
+    with open(file_path, "rb") as f:
+        while chunk := f.read(65536):
+            hasher.update(chunk)
+
+    return hasher.hexdigest()

lnclite-0.1.0/lnclite/utils/get_folder_fingerprint.py

@@ -0,0 +1,37 @@
+import os
+from pathlib import Path
+
+import xxhash
+
+
+def get_folder_fingerprint(target_path: Path | str, read_content: bool = False) -> str:
+    # Build a single rolling hash with xxh64.
+    folder_hash = xxhash.xxh64()
+
+    # Keep traversal order stable; otherwise the same tree can hash differently
+    # if os.walk yields files in a different order.
+    for root, dirs, files in os.walk(target_path):
+        for names in sorted(files):
+            file_path = os.path.join(root, names)
+
+            try:
+                # 1. Mix in relative path (detects moves/renames).
+                rel_path = os.path.relpath(file_path, target_path)
+                folder_hash.update(rel_path.encode())
+
+                # 2. Mix in file metadata — fast default.
+                # Use content hashing only if you need stronger accuracy
+                # and can pay the I/O cost.
+                if read_content:
+                    # Catches changes where mtime/size might not move (rare edge cases).
+                    with open(file_path, "rb") as f:
+                        folder_hash.update(f.read())
+                else:
+                    stat = os.stat(file_path)
+                    folder_hash.update(str(stat.st_mtime).encode())  # modification time
+                    folder_hash.update(str(stat.st_size).encode())  # file size
+
+            except (PermissionError, FileNotFoundError):
+                continue
+
+    return folder_hash.hexdigest()

lnclite-0.1.0/lnclite/utils/snowflake.py

@@ -0,0 +1,95 @@
+import hashlib
+import os
+import socket
+import threading
+import time
+from typing import Optional
+
+_global_generator: Optional["Snowflake"] = None
+
+
+class Snowflake:
+    def __init__(self, worker_id: int):
+        self.worker_id = worker_id
+        self.sequence = 0
+        self.last_timestamp = -1
+
+        self.twepoch = 1704067200000  # 2024-01-01
+
+        # Define the bit lengths of each part
+        self.worker_id_bits = 10
+        self.sequence_bits = 12
+
+        # Calculate the maximum value
+        self.max_worker_id = -1 ^ (-1 << self.worker_id_bits)
+
+        # Shift amounts
+        self.worker_id_shift = self.sequence_bits
+        self.timestamp_left_shift = self.sequence_bits + self.worker_id_bits
+        self.sequence_mask = -1 ^ (-1 << self.sequence_bits)
+
+        self.lock = threading.Lock()
+
+    def _get_timestamp(self):
+        return int(time.time() * 1000)
+
+    def generate(self) -> int:
+        with self.lock:
+            timestamp = self._get_timestamp()
+
+            if timestamp < self.last_timestamp:
+                raise Exception("Clock backward exception")
+
+            if timestamp == self.last_timestamp:
+                # Within the same millisecond, the sequence number increases
+                self.sequence = (self.sequence + 1) & self.sequence_mask
+                if self.sequence == 0:
+                    # If the sequence number is exhausted, wait for the next millisecond
+                    while timestamp <= self.last_timestamp:
+                        timestamp = self._get_timestamp()
+            else:
+                self.sequence = 0
+
+            self.last_timestamp = timestamp
+
+            # Combine the parts and perform bitwise left shift
+            new_id = (
+                ((timestamp - self.twepoch) << self.timestamp_left_shift)
+                | (self.worker_id << self.worker_id_shift)
+                | self.sequence
+            )
+
+            return new_id
+
+
+def get_valid_worker_id(max_bits: int = 10) -> int:
+    max_worker_id = (1 << max_bits) - 1  # Result of 1023
+    worker_id_str = os.getenv("WORKER_ID")
+
+    # Case 1 and Case 2: Environment variable has value
+    if worker_id_str:
+        try:
+            # Case 1: Standard numeric input
+            worker_id = int(worker_id_str)
+            # Use modulo to force the number to converge within the valid range
+            return abs(worker_id) % (max_worker_id + 1)
+        except ValueError:
+            # Case 2: Input non-numeric string (e.g. "app-worker-a")
+            # Use MD5 hash to convert the string to a large integer,
+            # then perform modulo convergence
+            hash_int = int(hashlib.md5(worker_id_str.encode("utf-8")).hexdigest(), 16)
+            return hash_int % (max_worker_id + 1)
+
+    # Case 3: No WORKER_ID is set (forced fallback)
+    # Get the hostname of the machine as the唯一性依據
+    hostname = socket.gethostname()
+    hash_int = int(hashlib.md5(hostname.encode("utf-8")).hexdigest(), 16)
+
+    return hash_int % (max_worker_id + 1)
+
+
+def generate_id() -> int:
+    global _global_generator
+    if _global_generator is None:
+        _global_generator = Snowflake(worker_id=get_valid_worker_id())
+    return _global_generator.generate()

lnclite-0.1.0/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+authors = [{ name = "Allen Chou", email = "f1470891079@gmail.com" }]
+dependencies = [
+  "lancedb",
+  "openai",
+  "openai-embeddings-model",
+  "paginatic",
+  "pydantic (>=2)",
+  "xxhash",
+]
+description = "Lite usages of lancedb."
+license = "MIT"
+license-files = ["LICENSE"]
+name = "lnclite"
+readme = "README.md"
+requires-python = ">=3.11,<4"
+version = "0.1.0"
+
+[project.urls]
+Homepage = "https://github.com/allen2c/lnclite"
+"PyPI" = "https://pypi.org/project/lnclite/"
+Repository = "https://github.com/allen2c/lnclite"
+
+[tool.poetry]
+packages = [{ include = "lnclite" }]
+
+[tool.poetry.group.dev.dependencies]
+black = { extras = ["jupyter"], version = "*" }
+isort = "*"
+lines-of-work = { git = "https://github.com/allen2c/lines-of-work.git" }
+logging-bullet-train = ">=0.4.0"
+mkdocs-material = "*"
+poetry-plugin-export = "*"
+pytest = "*"
+pytest-asyncio = "*"
+pytest-cov = "*"
+pytest-env = "*"
+pytest-xdist = "*"
+rich = "*"
+rich-color-support = "*"
+setuptools = "*"
+twine = "*"
+
+[tool.isort]
+profile = "black"
+
+[tool.black]
+target-version = ["py311"]
+
+[tool.flake8]
+ignore = ["E203", "E704", "W503"]
+max-line-length = 88
+
+[tool.pytest.ini_options]
+env = ["ENVIRONMENT=test", "PYTEST_IS_RUNNING=true"]
+
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core>=2.0.0,<3.0.0"]