swarmauri_vectorstore_fs 0.1.0__tar.gz

swarmauri_vectorstore_fs-0.1.0/PKG-INFO

@@ -0,0 +1,83 @@
+Metadata-Version: 2.4
+Name: swarmauri_vectorstore_fs
+Version: 0.1.0
+Summary: Swarmauri filesystem-aware BM25F vector store
+License-Expression: Apache-2.0
+Keywords: swarmauri,vectorstore,filesystem,bm25f,chunks,retrieval
+Author: Jacob Stewart
+Author-email: jacob@swarmauri.com
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Natural Language :: English
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Dist: rich (>=13.9.4)
+Requires-Dist: swarmauri_base
+Requires-Dist: swarmauri_core
+Requires-Dist: swarmauri_standard
+Description-Content-Type: text/markdown
+
+![Swarmauri Logo](https://github.com/swarmauri/swarmauri-sdk/blob/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri_brand_frag_light.png)
+
+<p align="center">
+    <a href="https://pypi.org/project/swarmauri_vectorstore_fs/">
+        <img src="https://img.shields.io/pypi/pyversions/swarmauri_vectorstore_fs" alt="PyPI - Python Version"/></a>
+    <a href="https://pypi.org/project/swarmauri_vectorstore_fs/">
+        <img src="https://img.shields.io/pypi/l/swarmauri_vectorstore_fs" alt="PyPI - License"/></a>
+    <a href="https://pypi.org/project/swarmauri_vectorstore_fs/">
+        <img src="https://img.shields.io/pypi/v/swarmauri_vectorstore_fs?label=swarmauri_vectorstore_fs&color=green" alt="PyPI - swarmauri_vectorstore_fs"/></a>
+</p>
+
+---
+
+# Swarmauri Vectorstore FS
+
+A Swarmauri community vector store that indexes filesystem trees for BM25F retrieval over file paths, file names, extensions, chunk identity, and file content.
+
+## Features
+
+- Filesystem-aware retrieval with weighted BM25F fields
+- Chunk, file, and chunk-plus-file indexing modes
+- Stable chunk identity metadata for global, path-level, and file-level chunk numbering
+- CLI for ad hoc lexical search over source trees and document corpora
+- No embedding vocabulary dependency for query handling
+
+## Installation
+
+```bash
+pip install swarmauri_vectorstore_fs
+```
+
+## Usage
+
+```python
+from swarmauri_vectorstore_fs import FsVectorStore
+
+store = FsVectorStore(root_path=".", mode="chunk")
+store.build_index()
+results = store.retrieve("vector store registration", top_k=3)
+
+for document in results:
+    print(document.id, document.metadata["relative_path"])
+```
+
+## CLI
+
+```bash
+fsvs --root . query --query "vector store registration" --top-k 5
+```
+
+To inspect a specific retrieved document:
+
+```bash
+fsvs --root . show --document-id <document-id>
+```
+
+## Want to help?
+
+If you want to contribute to swarmauri-sdk, read up on our [guidelines for contributing](https://github.com/swarmauri/swarmauri-sdk/blob/master/contributing.md).
+

swarmauri_vectorstore_fs-0.1.0/README.md

@@ -0,0 +1,59 @@
+![Swarmauri Logo](https://github.com/swarmauri/swarmauri-sdk/blob/3d4d1cfa949399d7019ae9d8f296afba773dfb7f/assets/swarmauri_brand_frag_light.png)
+
+<p align="center">
+    <a href="https://pypi.org/project/swarmauri_vectorstore_fs/">
+        <img src="https://img.shields.io/pypi/pyversions/swarmauri_vectorstore_fs" alt="PyPI - Python Version"/></a>
+    <a href="https://pypi.org/project/swarmauri_vectorstore_fs/">
+        <img src="https://img.shields.io/pypi/l/swarmauri_vectorstore_fs" alt="PyPI - License"/></a>
+    <a href="https://pypi.org/project/swarmauri_vectorstore_fs/">
+        <img src="https://img.shields.io/pypi/v/swarmauri_vectorstore_fs?label=swarmauri_vectorstore_fs&color=green" alt="PyPI - swarmauri_vectorstore_fs"/></a>
+</p>
+
+---
+
+# Swarmauri Vectorstore FS
+
+A Swarmauri community vector store that indexes filesystem trees for BM25F retrieval over file paths, file names, extensions, chunk identity, and file content.
+
+## Features
+
+- Filesystem-aware retrieval with weighted BM25F fields
+- Chunk, file, and chunk-plus-file indexing modes
+- Stable chunk identity metadata for global, path-level, and file-level chunk numbering
+- CLI for ad hoc lexical search over source trees and document corpora
+- No embedding vocabulary dependency for query handling
+
+## Installation
+
+```bash
+pip install swarmauri_vectorstore_fs
+```
+
+## Usage
+
+```python
+from swarmauri_vectorstore_fs import FsVectorStore
+
+store = FsVectorStore(root_path=".", mode="chunk")
+store.build_index()
+results = store.retrieve("vector store registration", top_k=3)
+
+for document in results:
+    print(document.id, document.metadata["relative_path"])
+```
+
+## CLI
+
+```bash
+fsvs --root . query --query "vector store registration" --top-k 5
+```
+
+To inspect a specific retrieved document:
+
+```bash
+fsvs --root . show --document-id <document-id>
+```
+
+## Want to help?
+
+If you want to contribute to swarmauri-sdk, read up on our [guidelines for contributing](https://github.com/swarmauri/swarmauri-sdk/blob/master/contributing.md).

swarmauri_vectorstore_fs-0.1.0/pyproject.toml

@@ -0,0 +1,82 @@
+[project]
+name = "swarmauri_vectorstore_fs"
+version = "0.1.0"
+description = "Swarmauri filesystem-aware BM25F vector store"
+license = "Apache-2.0"
+readme = "README.md"
+repository = "http://github.com/swarmauri/swarmauri-sdk"
+requires-python = ">=3.10,<3.13"
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Natural Language :: English",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development :: Libraries :: Application Frameworks",
+]
+authors = [{ name = "Jacob Stewart", email = "jacob@swarmauri.com" }]
+dependencies = [
+    "rich>=13.9.4",
+    "swarmauri_core",
+    "swarmauri_base",
+    "swarmauri_standard",
+]
+keywords = [
+    "swarmauri",
+    "vectorstore",
+    "filesystem",
+    "bm25f",
+    "chunks",
+    "retrieval",
+]
+
+[tool.uv.sources]
+swarmauri_core = { workspace = true }
+swarmauri_base = { workspace = true }
+swarmauri_standard = { workspace = true }
+
+[tool.pytest.ini_options]
+norecursedirs = ["combined", "scripts"]
+markers = [
+    "test: standard test",
+    "unit: Unit tests",
+    "i9n: Integration tests",
+    "r8n: Regression tests",
+    "timeout: mark test to timeout after X seconds",
+    "xpass: Expected passes",
+    "xfail: Expected failures",
+    "acceptance: Acceptance tests",
+    "perf: Performance tests that measure execution time and resource usage",
+]
+timeout = 300
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s [%(levelname)s] %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+asyncio_default_fixture_loop_scope = "function"
+
+[project.entry-points.'swarmauri.vector_stores']
+FsVectorStore = "swarmauri_vectorstore_fs.FsVectorStore:FsVectorStore"
+
+[project.scripts]
+fsvs = "swarmauri_vectorstore_fs.cli:main"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24.0",
+    "pytest-xdist>=3.6.1",
+    "pytest-json-report>=1.5.0",
+    "python-dotenv",
+    "requests>=2.32.3",
+    "flake8>=7.0",
+    "pytest-timeout>=2.3.1",
+    "ruff>=0.9.9",
+    "pytest-benchmark>=4.0.0",
+]

swarmauri_vectorstore_fs-0.1.0/swarmauri_vectorstore_fs/BM25FScorer.py

@@ -0,0 +1,142 @@
+import math
+import re
+from collections import Counter, defaultdict
+from typing import Dict, Iterable, List, Tuple
+
+
+DEFAULT_FIELD_WEIGHTS: Dict[str, float] = {
+    "file_name": 5.0,
+    "relative_path": 4.0,
+    "directory_path": 2.5,
+    "file_extension": 1.5,
+    "chunk_identity": 1.0,
+    "content": 1.0,
+}
+
+
+class BM25FScorer:
+    def __init__(
+        self,
+        field_weights: Dict[str, float] | None = None,
+        k1: float = 1.2,
+        b: float = 0.75,
+    ) -> None:
+        self.field_weights = dict(field_weights or DEFAULT_FIELD_WEIGHTS)
+        self.k1 = k1
+        self.b = b
+        self.document_fields: List[Dict[str, str]] = []
+        self._field_term_frequencies: List[Dict[str, Counter[str]]] = []
+        self._field_lengths: List[Dict[str, int]] = []
+        self._average_field_lengths: Dict[str, float] = {}
+        self._document_frequencies: Counter[str] = Counter()
+
+    @staticmethod
+    def tokenize(text: str) -> List[str]:
+        tokens: List[str] = []
+        for raw_token in re.findall(r"[A-Za-z0-9_]+", text.replace("\\", "/")):
+            token = raw_token.lower()
+            if token:
+                tokens.append(token)
+            parts = re.findall(
+                r"[A-Z]?[a-z]+|[A-Z]+(?=[A-Z]|$)|[0-9]+", raw_token.replace("_", " ")
+            )
+            for part in parts:
+                lowered = part.lower()
+                if lowered and lowered != token:
+                    tokens.append(lowered)
+        return tokens
+
+    def fit(self, document_fields: Iterable[Dict[str, str]]) -> None:
+        self.document_fields = [dict(fields) for fields in document_fields]
+        self._field_term_frequencies = []
+        self._field_lengths = []
+        self._average_field_lengths = {}
+        self._document_frequencies = Counter()
+
+        field_length_totals: Dict[str, int] = defaultdict(int)
+        for fields in self.document_fields:
+            doc_frequencies: Dict[str, Counter[str]] = {}
+            doc_lengths: Dict[str, int] = {}
+            doc_terms = set()
+            for field_name in self.field_weights:
+                tokens = self.tokenize(fields.get(field_name, ""))
+                frequencies = Counter(tokens)
+                doc_frequencies[field_name] = frequencies
+                doc_lengths[field_name] = len(tokens)
+                field_length_totals[field_name] += len(tokens)
+                doc_terms.update(frequencies)
+            self._field_term_frequencies.append(doc_frequencies)
+            self._field_lengths.append(doc_lengths)
+            for term in doc_terms:
+                self._document_frequencies[term] += 1
+
+        doc_count = max(1, len(self.document_fields))
+        self._average_field_lengths = {
+            field_name: field_length_totals[field_name] / doc_count
+            for field_name in self.field_weights
+        }
+
+    def search(self, query: str, top_k: int = 5) -> List[Tuple[int, float]]:
+        query_terms = self.tokenize(query)
+        if not query_terms or not self.document_fields:
+            return []
+
+        scores = [
+            (index, self._score_document(index, query_terms))
+            for index in range(len(self.document_fields))
+        ]
+        matches = [(index, score) for index, score in scores if score > 0.0]
+        matches.sort(key=lambda item: item[1], reverse=True)
+        return matches[:top_k]
+
+    def _score_document(self, document_index: int, query_terms: List[str]) -> float:
+        score = 0.0
+        for term in query_terms:
+            document_frequency = self._document_frequencies.get(term, 0)
+            if document_frequency == 0:
+                continue
+            weighted_frequency = self._weighted_term_frequency(document_index, term)
+            if weighted_frequency <= 0:
+                continue
+            score += self._idf(document_frequency) * (
+                (weighted_frequency * (self.k1 + 1.0)) / (self.k1 + weighted_frequency)
+            )
+        return score
+
+    def _weighted_term_frequency(self, document_index: int, term: str) -> float:
+        weighted_frequency = 0.0
+        for field_name, weight in self.field_weights.items():
+            field_frequency = self._field_term_frequencies[document_index][
+                field_name
+            ].get(term, 0)
+            if field_frequency == 0:
+                continue
+            field_length = self._field_lengths[document_index].get(field_name, 0)
+            average_length = self._average_field_lengths.get(field_name, 1.0) or 1.0
+            normalized_length = 1.0 - self.b + self.b * (field_length / average_length)
+            weighted_frequency += weight * (field_frequency / normalized_length)
+        return weighted_frequency
+
+    def _idf(self, document_frequency: int) -> float:
+        doc_count = len(self.document_fields)
+        return math.log(
+            1.0 + ((doc_count - document_frequency + 0.5) / (document_frequency + 0.5))
+        )
+
+    def to_dict(self) -> Dict:
+        return {
+            "field_weights": self.field_weights,
+            "k1": self.k1,
+            "b": self.b,
+            "document_fields": self.document_fields,
+        }
+
+    @classmethod
+    def from_dict(cls, payload: Dict) -> "BM25FScorer":
+        scorer = cls(
+            field_weights=payload.get("field_weights"),
+            k1=payload.get("k1", 1.2),
+            b=payload.get("b", 0.75),
+        )
+        scorer.fit(payload.get("document_fields", []))
+        return scorer

swarmauri_vectorstore_fs-0.1.0/swarmauri_vectorstore_fs/FsVectorStore.py

@@ -0,0 +1,414 @@
+import fnmatch
+import json
+from pathlib import Path
+from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
+
+from pydantic import Field, PrivateAttr
+from swarmauri_base.ComponentBase import ComponentBase
+from swarmauri_base.vector_stores.VectorStoreBase import VectorStoreBase
+from swarmauri_base.vector_stores.VectorStoreRetrieveMixin import (
+    VectorStoreRetrieveMixin,
+)
+from swarmauri_standard.documents.Document import Document
+
+from .BM25FScorer import BM25FScorer
+
+
+ModeLiteral = Literal["chunk", "file", "chunk_file"]
+DEFAULT_EXCLUDE_PATTERNS = (
+    ".git/**",
+    "**/.git/**",
+    "__pycache__/**",
+    "**/__pycache__/**",
+    ".venv/**",
+    "**/.venv/**",
+    "node_modules/**",
+    "**/node_modules/**",
+)
+
+
+@ComponentBase.register_type(VectorStoreBase, "FsVectorStore")
+class FsVectorStore(VectorStoreRetrieveMixin, VectorStoreBase):
+    type: Literal["FsVectorStore"] = "FsVectorStore"
+    root_path: Optional[str] = None
+    mode: ModeLiteral = "chunk"
+    chunk_size: int = 1200
+    chunk_overlap: int = 120
+    include: Tuple[str, ...] = Field(default=())
+    exclude: Tuple[str, ...] = Field(default=())
+    max_file_size: int = 1_000_000
+    auto_index: bool = False
+    index_metadata: Dict[str, int] = Field(default_factory=dict)
+
+    _scorer = PrivateAttr()
+    _document_map = PrivateAttr()
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._scorer = BM25FScorer()
+        self._document_map: Dict[str, Document] = {}
+        self.documents = []
+        if self.auto_index and self.root_path:
+            self.build_index()
+
+    def add_document(self, document: Document) -> None:
+        self._document_map[document.id] = document
+        self.documents = list(self._document_map.values())
+        self._reindex_documents()
+
+    def add_documents(self, documents: List[Document]) -> None:
+        for document in documents:
+            self._document_map[document.id] = document
+        self.documents = list(self._document_map.values())
+        self._reindex_documents()
+
+    def get_document(self, id: str) -> Union[Document, None]:
+        return self._document_map.get(id)
+
+    def get_all_documents(self) -> List[Document]:
+        return list(self.documents)
+
+    def update_document(self, id: str, updated_document: Document) -> None:
+        if id not in self._document_map:
+            raise KeyError(f"Document '{id}' not found")
+        self._document_map[id] = updated_document
+        self.documents = list(self._document_map.values())
+        self._reindex_documents()
+
+    def delete_document(self, id: str) -> None:
+        self._document_map.pop(id, None)
+        self.documents = list(self._document_map.values())
+        self._reindex_documents()
+
+    def clear_documents(self) -> None:
+        self.documents = []
+        self._document_map = {}
+        self._scorer = BM25FScorer()
+
+    def build_index(self) -> None:
+        root = self._validate_root()
+        documents: List[Document] = []
+        skip_counts = {
+            "binary": 0,
+            "oversized": 0,
+            "unreadable": 0,
+            "excluded": 0,
+        }
+        global_index = 0
+
+        for file_index, path in enumerate(
+            self._iter_indexable_files(root, skip_counts)
+        ):
+            relative_path = path.relative_to(root).as_posix()
+            text = self._read_text(path, skip_counts)
+            if text is None:
+                continue
+
+            chunks = self._chunk_text(text)
+            if self.mode in {"chunk", "chunk_file"}:
+                for chunk_file_index, (chunk, start_line, end_line) in enumerate(
+                    chunks
+                ):
+                    document = self._build_document(
+                        relative_path=relative_path,
+                        file_index=file_index,
+                        chunk_global_index=global_index,
+                        chunk_path_index=chunk_file_index,
+                        chunk_file_index=chunk_file_index,
+                        start_line=start_line,
+                        end_line=end_line,
+                        document_kind="chunk",
+                        content=chunk,
+                    )
+                    documents.append(document)
+                    global_index += 1
+
+            if self.mode in {"file", "chunk_file"}:
+                document = self._build_document(
+                    relative_path=relative_path,
+                    file_index=file_index,
+                    chunk_global_index=global_index,
+                    chunk_path_index=len(chunks),
+                    chunk_file_index=-1,
+                    start_line=1,
+                    end_line=self._line_count(text),
+                    document_kind="file",
+                    content=text,
+                )
+                documents.append(document)
+                global_index += 1
+
+        self.index_metadata = {
+            **skip_counts,
+            "indexed_documents": len(documents),
+        }
+        self._document_map = {document.id: document for document in documents}
+        self.documents = documents
+        self._reindex_documents()
+
+    def refresh_index(self) -> None:
+        self.build_index()
+
+    def retrieve(self, query: str, top_k: int = 5) -> List[Document]:
+        return [document for document, _score in self.search(query, top_k=top_k)]
+
+    def query(self, text: str, top_k: int = 5) -> List[Document]:
+        return self.retrieve(text, top_k=top_k)
+
+    def search(self, query: str, top_k: int = 5) -> List[Tuple[Document, float]]:
+        if not self.documents and self.root_path:
+            self.build_index()
+        results = self._scorer.search(query, top_k=top_k)
+        return [(self.documents[index], score) for index, score in results]
+
+    def save_store(self, directory_path: str) -> None:
+        output_dir = Path(directory_path)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        payload = {
+            "root_path": self.root_path,
+            "mode": self.mode,
+            "chunk_size": self.chunk_size,
+            "chunk_overlap": self.chunk_overlap,
+            "include": list(self.include),
+            "exclude": list(self.exclude),
+            "max_file_size": self.max_file_size,
+            "index_metadata": self.index_metadata,
+            "bm25f": self._scorer.to_dict(),
+            "documents": [
+                document.model_dump(mode="json") for document in self.documents
+            ],
+        }
+        (output_dir / "fs_vectorstore.json").write_text(
+            json.dumps(payload, indent=2), encoding="utf-8"
+        )
+
+    def load_store(self, directory_path: str) -> None:
+        payload = json.loads(
+            (Path(directory_path) / "fs_vectorstore.json").read_text(encoding="utf-8")
+        )
+        self.root_path = payload.get("root_path")
+        self.mode = payload.get("mode", "chunk")
+        self.chunk_size = payload.get("chunk_size", 1200)
+        self.chunk_overlap = payload.get("chunk_overlap", 120)
+        self.include = tuple(payload.get("include", []))
+        self.exclude = tuple(payload.get("exclude", []))
+        self.max_file_size = payload.get("max_file_size", 1_000_000)
+        self.index_metadata = payload.get("index_metadata", {})
+        self.documents = [
+            Document.model_validate(document) for document in payload["documents"]
+        ]
+        self._document_map = {document.id: document for document in self.documents}
+        self._scorer = BM25FScorer.from_dict(payload.get("bm25f", {}))
+        if not self._scorer.document_fields:
+            self._reindex_documents()
+
+    def _reindex_documents(self) -> None:
+        self._scorer = BM25FScorer()
+        self._scorer.fit(
+            [self._fields_for_document(document) for document in self.documents]
+        )
+
+    def _validate_root(self) -> Path:
+        if not self.root_path:
+            raise ValueError("root_path is required to build the filesystem index")
+        root = Path(self.root_path)
+        if not root.exists():
+            raise FileNotFoundError(f"Root path '{self.root_path}' does not exist")
+        if not root.is_dir():
+            raise NotADirectoryError(f"Root path '{self.root_path}' is not a directory")
+        if self.chunk_size <= 0:
+            raise ValueError("chunk_size must be greater than zero")
+        if self.chunk_overlap < 0:
+            raise ValueError("chunk_overlap must be zero or greater")
+        if self.chunk_overlap >= self.chunk_size:
+            raise ValueError("chunk_overlap must be smaller than chunk_size")
+        return root
+
+    def _iter_indexable_files(
+        self, root: Path, skip_counts: Dict[str, int]
+    ) -> Iterable[Path]:
+        for path in sorted(root.rglob("*")):
+            if not path.is_file():
+                continue
+            relative_path = path.relative_to(root).as_posix()
+            if self.include and not self._matches_any(relative_path, self.include):
+                skip_counts["excluded"] += 1
+                continue
+            if self._matches_any(
+                relative_path, (*DEFAULT_EXCLUDE_PATTERNS, *self.exclude)
+            ):
+                skip_counts["excluded"] += 1
+                continue
+            yield path
+
+    def _read_text(self, path: Path, skip_counts: Dict[str, int]) -> Optional[str]:
+        try:
+            size = path.stat().st_size
+            if size > self.max_file_size:
+                skip_counts["oversized"] += 1
+                return None
+            data = path.read_bytes()
+        except OSError:
+            skip_counts["unreadable"] += 1
+            return None
+
+        if b"\x00" in data:
+            skip_counts["binary"] += 1
+            return None
+
+        try:
+            return data.decode("utf-8")
+        except UnicodeDecodeError:
+            skip_counts["binary"] += 1
+            return None
+
+    def _chunk_text(self, text: str) -> List[Tuple[str, int, int]]:
+        if not text:
+            return [("", 1, 1)]
+
+        chunks: List[Tuple[str, int, int]] = []
+        start = 0
+        text_length = len(text)
+        while start < text_length:
+            end = min(text_length, start + self.chunk_size)
+            chunk = text[start:end]
+            start_line = text.count("\n", 0, start) + 1
+            end_line = text.count("\n", 0, end) + 1
+            chunks.append((chunk, start_line, end_line))
+            if end == text_length:
+                break
+            start = max(end - self.chunk_overlap, start + 1)
+        return chunks
+
+    def _build_document(
+        self,
+        *,
+        relative_path: str,
+        file_index: int,
+        chunk_global_index: int,
+        chunk_path_index: int,
+        chunk_file_index: int,
+        start_line: int,
+        end_line: int,
+        document_kind: Literal["chunk", "file"],
+        content: str,
+    ) -> Document:
+        path = Path(relative_path)
+        file_name = path.name
+        file_extension = path.suffix
+        directory_path = path.parent.as_posix() if path.parent.as_posix() != "." else ""
+        bm25_fields = self._build_bm25_fields(
+            relative_path=relative_path,
+            directory_path=directory_path,
+            file_name=file_name,
+            file_extension=file_extension,
+            file_index=file_index,
+            chunk_global_index=chunk_global_index,
+            chunk_path_index=chunk_path_index,
+            chunk_file_index=chunk_file_index,
+            content=content,
+        )
+        metadata = {
+            "relative_path": relative_path,
+            "file_index": file_index,
+            "file_name": file_name,
+            "file_extension": file_extension,
+            "chunk_global_index": chunk_global_index,
+            "chunk_path_index": chunk_path_index,
+            "chunk_file_index": chunk_file_index,
+            "start_line": start_line,
+            "end_line": end_line,
+            "document_kind": document_kind,
+            "bm25_fields": bm25_fields,
+        }
+        return Document(
+            id=f"fs:{file_index}:{document_kind}:{chunk_path_index}",
+            content=self._composite_content(metadata, content),
+            metadata=metadata,
+        )
+
+    def _build_bm25_fields(
+        self,
+        *,
+        relative_path: str,
+        directory_path: str,
+        file_name: str,
+        file_extension: str,
+        file_index: int,
+        chunk_global_index: int,
+        chunk_path_index: int,
+        chunk_file_index: int,
+        content: str,
+    ) -> Dict[str, str]:
+        chunk_identity = " ".join(
+            [
+                f"file_index {file_index}",
+                f"file_{file_index}",
+                f"chunk_global_index {chunk_global_index}",
+                f"chunk_global_{chunk_global_index}",
+                f"chunk_path_index {chunk_path_index}",
+                f"chunk_path_{chunk_path_index}",
+                f"chunk_file_index {chunk_file_index}",
+                f"chunk_file_{chunk_file_index}",
+            ]
+        )
+        return {
+            "file_name": file_name,
+            "relative_path": relative_path,
+            "directory_path": directory_path,
+            "file_extension": file_extension,
+            "chunk_identity": chunk_identity,
+            "content": content,
+        }
+
+    def _fields_for_document(self, document: Document) -> Dict[str, str]:
+        fields = document.metadata.get("bm25_fields")
+        if isinstance(fields, dict):
+            return {field: str(value) for field, value in fields.items()}
+        return {
+            "file_name": str(document.metadata.get("file_name", "")),
+            "relative_path": str(document.metadata.get("relative_path", "")),
+            "directory_path": str(
+                Path(str(document.metadata.get("relative_path", ""))).parent
+            ),
+            "file_extension": str(document.metadata.get("file_extension", "")),
+            "chunk_identity": " ".join(
+                str(document.metadata.get(key, ""))
+                for key in (
+                    "file_index",
+                    "chunk_global_index",
+                    "chunk_path_index",
+                    "chunk_file_index",
+                )
+            ),
+            "content": document.content,
+        }
+
+    def _composite_content(self, metadata: Dict, content: str) -> str:
+        return "\n".join(
+            [
+                f"file_index: {metadata['file_index']}",
+                f"file_path: {metadata['relative_path']}",
+                f"file_name: {metadata['file_name']}",
+                f"file_extension: {metadata['file_extension']}",
+                f"chunk_global_index: {metadata['chunk_global_index']}",
+                f"chunk_path_index: {metadata['chunk_path_index']}",
+                f"chunk_file_index: {metadata['chunk_file_index']}",
+                f"document_kind: {metadata['document_kind']}",
+                "content:",
+                content,
+            ]
+        )
+
+    def _matches_any(self, relative_path: str, patterns: Tuple[str, ...]) -> bool:
+        path = relative_path.replace("\\", "/")
+        name = Path(path).name
+        return any(
+            fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(name, pattern)
+            for pattern in patterns
+        )
+
+    def _line_count(self, text: str) -> int:
+        if not text:
+            return 1
+        return text.count("\n") + (0 if text.endswith("\n") else 1)

swarmauri_vectorstore_fs-0.1.0/swarmauri_vectorstore_fs/__init__.py

@@ -0,0 +1,14 @@
+from .BM25FScorer import BM25FScorer
+from .FsVectorStore import FsVectorStore
+
+__all__ = ["BM25FScorer", "FsVectorStore"]
+
+try:
+    from importlib.metadata import PackageNotFoundError, version
+except ImportError:
+    from importlib_metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("swarmauri_vectorstore_fs")
+except PackageNotFoundError:
+    __version__ = "0.0.0"

swarmauri_vectorstore_fs-0.1.0/swarmauri_vectorstore_fs/cli.py

@@ -0,0 +1,171 @@
+import argparse
+import json
+import logging
+from typing import Iterable, Optional
+
+from rich.console import Console
+from rich.logging import RichHandler
+from rich.table import Table
+
+from .FsVectorStore import FsVectorStore
+
+
+LOGGER = logging.getLogger("swarmauri.fs_vectorstore")
+CONSOLE = Console()
+
+
+def configure_logging(verbose: bool = False) -> None:
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(
+        level=level,
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, markup=True)],
+        force=True,
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="BM25F retrieval over filesystem files and chunks."
+    )
+    parser.add_argument("--root", default=".", help="Root directory to index")
+    parser.add_argument(
+        "--mode",
+        choices=["chunk", "file", "chunk_file"],
+        default="chunk",
+        help="Indexing mode",
+    )
+    parser.add_argument(
+        "--chunk-size", type=int, default=1200, help="Chunk size in characters"
+    )
+    parser.add_argument(
+        "--chunk-overlap", type=int, default=120, help="Chunk overlap in characters"
+    )
+    parser.add_argument(
+        "--include", action="append", dest="include", help="Glob to include"
+    )
+    parser.add_argument(
+        "--exclude", action="append", dest="exclude", help="Glob to exclude"
+    )
+    parser.add_argument(
+        "--max-file-size",
+        type=int,
+        default=1_000_000,
+        help="Maximum file size in bytes to index",
+    )
+    parser.add_argument("--verbose", action="store_true", help="Enable debug logging")
+
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    query_parser = subparsers.add_parser(
+        "query", help="Query indexed filesystem documents"
+    )
+    query_parser.add_argument("--query", required=True, help="Query text")
+    query_parser.add_argument(
+        "--top-k", type=int, default=5, help="Number of hits to return"
+    )
+    query_parser.add_argument("--json", action="store_true", help="Emit JSON output")
+
+    show_parser = subparsers.add_parser("show", help="Print a document by id")
+    show_parser.add_argument(
+        "--document-id", required=True, help="Document id to display"
+    )
+
+    return parser
+
+
+def build_store(args: argparse.Namespace) -> FsVectorStore:
+    store = FsVectorStore(
+        root_path=args.root,
+        mode=args.mode,
+        chunk_size=args.chunk_size,
+        chunk_overlap=args.chunk_overlap,
+        include=tuple(args.include or ()),
+        exclude=tuple(args.exclude or ()),
+        max_file_size=args.max_file_size,
+    )
+    LOGGER.info(
+        "Building filesystem BM25F index [bold cyan](root=%s, mode=%s)[/bold cyan]",
+        args.root,
+        args.mode,
+    )
+    store.build_index()
+    LOGGER.info("Indexed [bold green]%s[/bold green] documents", len(store.documents))
+    return store
+
+
+def render_query_results(
+    store: FsVectorStore, query: str, top_k: int, as_json: bool
+) -> int:
+    LOGGER.info(
+        "Running BM25F retrieval for query [bold yellow]%s[/bold yellow]", query
+    )
+    results = store.search(query, top_k=top_k)
+    if not results:
+        LOGGER.warning(
+            "No lexical matches found for query [bold yellow]%s[/bold yellow]", query
+        )
+
+    if as_json:
+        payload = [
+            {
+                "id": document.id,
+                "score": score,
+                "metadata": document.metadata,
+                "content": document.content,
+            }
+            for document, score in results
+        ]
+        CONSOLE.print_json(json.dumps(payload))
+        return 0
+
+    table = Table(title=f"Fs Vector Store Results ({len(results)} hits)")
+    table.add_column("Document ID", style="cyan", overflow="fold")
+    table.add_column("Score", style="green")
+    table.add_column("Kind", style="magenta")
+    table.add_column("Path", style="white", overflow="fold")
+    table.add_column("Lines", style="blue")
+
+    for document, score in results:
+        table.add_row(
+            document.id,
+            f"{score:.4f}",
+            str(document.metadata.get("document_kind", "")),
+            str(document.metadata.get("relative_path", "")),
+            f"{document.metadata.get('start_line')}:{document.metadata.get('end_line')}",
+        )
+    CONSOLE.print(table)
+    return 0
+
+
+def render_document(store: FsVectorStore, document_id: str) -> int:
+    document = store.get_document(document_id)
+    if not document:
+        LOGGER.error("Document [bold red]%s[/bold red] was not found", document_id)
+        return 1
+
+    LOGGER.info("Printing document [bold cyan]%s[/bold cyan]", document_id)
+    CONSOLE.rule(f"[bold blue]{document_id}")
+    CONSOLE.print_json(json.dumps(document.metadata))
+    CONSOLE.print(document.content)
+    return 0
+
+
+def main(argv: Optional[Iterable[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(list(argv) if argv is not None else None)
+    configure_logging(verbose=args.verbose)
+    store = build_store(args)
+
+    if args.command == "query":
+        return render_query_results(store, args.query, args.top_k, args.json)
+    if args.command == "show":
+        return render_document(store, args.document_id)
+
+    parser.error(f"Unknown command: {args.command}")
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())