hecvec 0.1.0__py3-none-any.whl

hecvec/__init__.py

@@ -0,0 +1,34 @@
+"""
+HecVec: modular library — listdir, read, chunk (token/recursive), embed, Chroma. No API.
+"""
+from hecvec.chunking import chunk_documents, chunk_text
+from hecvec.chroma_client import add_documents, get_client, get_or_create_collection
+from hecvec.chroma_list import list_collections
+from hecvec.embeddings import embed_texts
+from hecvec.env import load_dotenv_if_available, load_openai_key
+from hecvec.listdir import ALLOWED_EXTENSIONS, ListDir, ListDirTextFiles
+from hecvec.pipeline import HecVec
+from hecvec.reading import ReadText
+from hecvec.token_splitter import token_chunk_documents, token_chunk_text
+
+__all__ = [
+    "ALLOWED_EXTENSIONS",
+    "HecVec",
+    "ListDir",
+    "ListDirTextFiles",
+    "ReadText",
+    "add_documents",
+    "chunk_documents",
+    "chunk_text",
+    "embed_texts",
+    "get_client",
+    "get_or_create_collection",
+    "list_collections",
+    "load_dotenv_if_available",
+    "load_openai_key",
+    "token_chunk_documents",
+    "token_chunk_text",
+    "__version__",
+]
+
+__version__ = "0.1.0"

hecvec/chroma_client.py

@@ -0,0 +1,58 @@
+"""
+Chroma client and collection operations. One module = one responsibility: connect and add documents.
+Requires: pip install hecvec[chroma] (chromadb).
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import chromadb
+
+DEFAULT_HOST = "localhost"
+DEFAULT_PORT = 8000
+
+
+def get_client(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT):
+    """Return a Chroma HTTP client, or an in-memory client if the server is not reachable."""
+    import chromadb
+    try:
+        return chromadb.HttpClient(host=host, port=port)
+    except Exception:
+        return chromadb.EphemeralClient()
+
+
+def get_or_create_collection(
+    client: "chromadb.HttpClient",
+    name: str,
+    metadata: dict | None = None,
+):
+    """Get or create a collection (cosine similarity by default)."""
+    if metadata is None:
+        metadata = {"hnsw:space": "cosine"}
+    return client.get_or_create_collection(name=name, metadata=metadata)
+
+
+def add_documents(
+    client: "chromadb.HttpClient",
+    collection_name: str,
+    ids: list[str],
+    embeddings: list[list[float]],
+    documents: list[str],
+) -> None:
+    """
+    Add documents to a collection. If dimension mismatch, deletes and recreates the collection.
+    """
+    import chromadb
+    coll = get_or_create_collection(client, collection_name)
+    try:
+        coll.add(ids=ids, embeddings=embeddings, documents=documents)
+    except chromadb.errors.InvalidArgumentError as e:
+        if "dimension" not in str(e).lower():
+            raise
+        client.delete_collection(name=collection_name)
+        coll = client.create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"},
+        )
+        coll.add(ids=ids, embeddings=embeddings, documents=documents)

hecvec/chroma_list.py

@@ -0,0 +1,27 @@
+"""
+List Chroma collections. One module = one responsibility: inspect collections on a Chroma server.
+Requires: pip install hecvec[chroma] (chromadb).
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import chromadb
+
+DEFAULT_HOST = "localhost"
+DEFAULT_PORT = 8000
+
+
+def list_collections(
+    host: str = DEFAULT_HOST,
+    port: int = DEFAULT_PORT,
+) -> list[tuple[str, int]]:
+    """
+    List all collection names and their document counts on a Chroma server.
+    Returns [(name, count), ...].
+    """
+    import chromadb
+    client = chromadb.HttpClient(host=host, port=port)
+    collections = client.list_collections()
+    return [(c.name, c.count()) for c in collections]

hecvec/chunking.py

@@ -0,0 +1,69 @@
+"""
+Chunk text documents. Requires optional dependency: pip install hecvec[chunk]
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+_DEFAULT_CHUNK_SIZE = 400
+_DEFAULT_CHUNK_OVERLAP = 0
+_DEFAULT_SEPARATORS = ["\n\n\n", "\n\n", "\n", ". ", " ", ""]
+
+
+def _get_splitter(
+    chunk_size: int = _DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = _DEFAULT_CHUNK_OVERLAP,
+    separators: list[str] | None = None,
+) -> "RecursiveCharacterTextSplitter":
+    try:
+        from langchain_text_splitters import RecursiveCharacterTextSplitter
+    except ImportError as e:
+        raise ImportError(
+            "Chunking requires langchain-text-splitters. Install with: pip install hecvec[chunk]"
+        ) from e
+    return RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=separators or _DEFAULT_SEPARATORS,
+    )
+
+
+def chunk_text(
+    text: str,
+    chunk_size: int = _DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = _DEFAULT_CHUNK_OVERLAP,
+    separators: list[str] | None = None,
+) -> list[str]:
+    """
+    Split a single document text into chunks.
+
+    Requires optional dependency: pip install hecvec[chunk]
+    """
+    splitter = _get_splitter(chunk_size, chunk_overlap, separators)
+    return splitter.split_text(text)
+
+
+def chunk_documents(
+    path_and_texts: list[tuple[str | Path, str]],
+    chunk_size: int = _DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = _DEFAULT_CHUNK_OVERLAP,
+    separators: list[str] | None = None,
+) -> list[dict]:
+    """
+    Chunk multiple documents. Each input is (path, text).
+    Returns a list of dicts: {"path": str, "chunk_index": int, "content": str}.
+
+    Requires optional dependency: pip install hecvec[chunk]
+    """
+    splitter = _get_splitter(chunk_size, chunk_overlap, separators)
+    out = []
+    for path, text in path_and_texts:
+        path_str = str(Path(path).resolve())
+        chunks = splitter.split_text(text)
+        for i, content in enumerate(chunks):
+            out.append({"path": path_str, "chunk_index": i, "content": content})
+    return out

hecvec/cli.py

@@ -0,0 +1,27 @@
+"""CLI for listing directory contents under a root."""
+import sys
+from pathlib import Path
+
+from hecvec import ListDir
+
+
+def main() -> None:
+    root = Path.cwd()
+    path = "."
+    if len(sys.argv) >= 2:
+        path = sys.argv[1]
+    if len(sys.argv) >= 3:
+        root = Path(sys.argv[2]).resolve()
+
+    lister = ListDir(root=root)
+    entries = lister.listdir(path)
+    print(f"Contenido de: {path}\n")
+    for rel in entries:
+        full = lister.root / rel
+        prefix = "[DIR] " if full.is_dir() else "      "
+        print(f"  {prefix}{rel}")
+    print(f"\nTotal: {len(entries)} entradas")
+
+
+if __name__ == "__main__":
+    main()

hecvec/embeddings.py

@@ -0,0 +1,39 @@
+"""
+OpenAI embeddings. One module = one responsibility: turn text list into embedding vectors.
+Requires: pip install hecvec[chroma] (openai).
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from openai import OpenAI
+
+DEFAULT_MODEL = "text-embedding-3-small"
+DEFAULT_BATCH_SIZE = 100
+
+
+def _get_openai_client(api_key: str | None = None) -> "OpenAI":
+    from openai import OpenAI
+    if not api_key:
+        raise ValueError("api_key is required for embeddings")
+    return OpenAI(api_key=api_key)
+
+
+def embed_texts(
+    texts: list[str],
+    api_key: str,
+    model: str = DEFAULT_MODEL,
+    batch_size: int = DEFAULT_BATCH_SIZE,
+) -> list[list[float]]:
+    """
+    Embed a list of text strings with OpenAI.
+    Returns a list of embedding vectors (same order as texts).
+    """
+    client = _get_openai_client(api_key)
+    out: list[list[float]] = []
+    for start in range(0, len(texts), batch_size):
+        batch = texts[start : start + batch_size]
+        response = client.embeddings.create(model=model, input=batch)
+        out.extend([item.embedding for item in response.data])
+    return out

hecvec/env.py

@@ -0,0 +1,26 @@
+"""
+Load environment and secrets (e.g. OPENAI_API_KEY from .env).
+Use this module to centralize dotenv and env access.
+"""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def load_dotenv_if_available(dotenv_path: str | Path | None = None) -> None:
+    """Load .env into os.environ if python-dotenv is installed. No-op otherwise."""
+    try:
+        from dotenv import load_dotenv
+        load_dotenv(dotenv_path)
+    except ImportError:
+        pass
+
+
+def load_openai_key(dotenv_path: str | Path | None = None) -> str | None:
+    """
+    Load OPENAI_API_KEY from the environment.
+    If dotenv_path is given (or python-dotenv is available), loads .env first.
+    """
+    load_dotenv_if_available(dotenv_path)
+    return os.environ.get("OPENAI_API_KEY")

hecvec/hecvec.py

@@ -0,0 +1,6 @@
+"""
+HecVec pipeline entrypoint. Re-exports HecVec from pipeline for backward compatibility.
+"""
+from hecvec.pipeline import HecVec
+
+__all__ = ["HecVec"]

hecvec/listdir.py

@@ -0,0 +1,126 @@
+"""Directory listing with safe path resolution under a root."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+ALLOWED_EXTENSIONS = (".txt", ".md")
+
+
+class ListDir:
+    """
+    Lists directory contents with safe path resolution. All paths are
+    restricted under `root`. Returns a single list of relative path strings.
+    """
+
+    def __init__(self, root: str | Path):
+        self.root = Path(root).resolve()
+        if not self.root.is_dir():
+            raise ValueError(f"root must be an existing directory: {self.root}")
+
+    def _resolve(self, path: str | Path) -> Path:
+        """Resolve path under root; raise ValueError if it escapes root."""
+        p = Path(path)
+        if not p.is_absolute():
+            p = self.root / p
+        resolved = p.resolve()
+        try:
+            resolved.relative_to(self.root)
+        except ValueError:
+            raise ValueError(f"Path escapes allowed root: {path!r} -> {resolved}")
+        return resolved
+
+    def listdir(self, path: str | Path = ".") -> list[str]:
+        """
+        List entries in the given path (under root). Returns a sorted list
+        of relative path strings (dirs first, then files, by name).
+        """
+        target = self._resolve(path)
+        if not target.is_dir():
+            raise ValueError(f"Not a directory: {target}")
+        out = []
+        for entry in target.iterdir():
+            try:
+                rel = entry.relative_to(self.root)
+            except ValueError:
+                continue
+            out.append(str(rel))
+        return sorted(out, key=lambda x: (not (self.root / x).is_dir(), x.lower()))
+
+    def listdir_recursive(
+        self,
+        path: str | Path = ".",
+        max_depth: int | None = None,
+    ) -> list[str]:
+        """
+        List all entries under path recursively. Returns a flat sorted list
+        of relative path strings.
+        """
+        target = self._resolve(path)
+        if not target.is_dir():
+            raise ValueError(f"Not a directory: {target}")
+        out = []
+        for root_dir, dirs, files in os.walk(target, topdown=True):
+            root_path = Path(root_dir)
+            try:
+                rel_root = root_path.relative_to(self.root)
+            except ValueError:
+                continue
+            depth = len(rel_root.parts) if rel_root != Path(".") else 0
+            if max_depth is not None and depth > max_depth:
+                dirs.clear()
+                continue
+            for d in sorted(dirs):
+                p = root_path / d
+                try:
+                    rel = p.relative_to(self.root)
+                except ValueError:
+                    continue
+                out.append(str(rel))
+            for f in sorted(files):
+                p = root_path / f
+                try:
+                    rel = p.relative_to(self.root)
+                except ValueError:
+                    continue
+                out.append(str(rel))
+        return sorted(out)
+
+
+class ListDirTextFiles(ListDir):
+    """
+    Child of ListDir. Takes the list of relative paths from the parent and
+    returns only .txt and .md files as full paths; filters out all other
+    extensions.
+    """
+
+    def __init__(self, root: str | Path, allowed_extensions: tuple[str, ...] = ALLOWED_EXTENSIONS):
+        super().__init__(root)
+        self.allowed_extensions = allowed_extensions
+
+    def filter_txt_md(self, relative_paths: list[str]) -> list[Path]:
+        """
+        From a list of relative path strings (e.g. from listdir/listdir_recursive),
+        build full paths and return only those that are files with .txt or .md.
+        """
+        result = []
+        for rel in relative_paths:
+            full = self.root / rel
+            if full.is_file() and full.suffix.lower() in self.allowed_extensions:
+                result.append(full.resolve())
+        return sorted(result)
+
+    def listdir_txt_md(self, path: str | Path = ".") -> list[Path]:
+        """List only .txt and .md files under path (one level)."""
+        paths = self.listdir(path)
+        return self.filter_txt_md(paths)
+
+    def listdir_recursive_txt_md(
+        self,
+        path: str | Path = ".",
+        max_depth: int | None = None,
+    ) -> list[Path]:
+        """List only .txt and .md files under path recursively."""
+        paths = self.listdir_recursive(path=path, max_depth=max_depth)
+        return self.filter_txt_md(paths)

hecvec/pipeline.py

@@ -0,0 +1,187 @@
+"""
+Pipeline: composes listdir → read → token-chunk → embed → Chroma.
+One module = one responsibility: orchestrate the full slice() flow.
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from hecvec.chroma_client import add_documents, get_client
+from hecvec.embeddings import embed_texts
+from hecvec.env import load_openai_key
+from hecvec.listdir import ListDirTextFiles
+from hecvec.reading import ReadText
+from hecvec.token_splitter import token_chunk_documents
+
+logger = logging.getLogger(__name__)
+
+
+def _check_chroma_deps() -> None:
+    try:
+        import chromadb  # noqa: F401
+        from langchain_text_splitters import TokenTextSplitter  # noqa: F401
+        from openai import OpenAI  # noqa: F401
+    except ImportError as e:
+        raise ImportError(
+            "Chroma pipeline requires: pip install hecvec[chroma]"
+        ) from e
+
+
+class HecVec:
+    """
+    Library-only pipeline. No API.
+    Call slice(path=...) to: list path → filter .txt/.md → token-chunk → push to Chroma.
+    """
+
+    def __init__(
+        self,
+        *,
+        root: str | Path | None = None,
+        collection_name: str = "hecvec",
+        chroma_host: str = "localhost",
+        chroma_port: int = 8000,
+        embedding_model: str = "text-embedding-3-small",
+        chunk_size: int = 200,
+        chunk_overlap: int = 0,
+        encoding_name: str = "cl100k_base",
+        batch_size: int = 100,
+        openai_api_key: str | None = None,
+        dotenv_path: str | Path | None = None,
+    ):
+        self.root = Path(root).resolve() if root else Path.cwd()
+        self.collection_name = collection_name
+        self.chroma_host = chroma_host
+        self.chroma_port = chroma_port
+        self.embedding_model = embedding_model
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.encoding_name = encoding_name
+        self.batch_size = batch_size
+        self._dotenv_path = dotenv_path
+        self._openai_api_key = openai_api_key or load_openai_key(dotenv_path)
+
+    def slice(
+        self,
+        path: str | Path,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """Run the pipeline for this instance (uses instance config)."""
+        return HecVec.slice(
+            path,
+            root=kwargs.pop("root", self.root),
+            collection_name=kwargs.pop("collection_name", self.collection_name),
+            chroma_host=kwargs.pop("chroma_host", self.chroma_host),
+            chroma_port=kwargs.pop("chroma_port", self.chroma_port),
+            embedding_model=kwargs.pop("embedding_model", self.embedding_model),
+            chunk_size=kwargs.pop("chunk_size", self.chunk_size),
+            chunk_overlap=kwargs.pop("chunk_overlap", self.chunk_overlap),
+            encoding_name=kwargs.pop("encoding_name", self.encoding_name),
+            batch_size=kwargs.pop("batch_size", self.batch_size),
+            openai_api_key=kwargs.pop("openai_api_key", self._openai_api_key),
+            dotenv_path=kwargs.pop("dotenv_path", self._dotenv_path),
+            **kwargs,
+        )
+
+    @classmethod
+    def slice(
+        cls,
+        path: str | Path,
+        *,
+        root: str | Path | None = None,
+        collection_name: str = "hecvec",
+        chroma_host: str = "localhost",
+        chroma_port: int = 8000,
+        embedding_model: str = "text-embedding-3-small",
+        chunk_size: int = 200,
+        chunk_overlap: int = 0,
+        encoding_name: str = "cl100k_base",
+        batch_size: int = 100,
+        openai_api_key: str | None = None,
+        dotenv_path: str | Path | None = None,
+        **kwargs: Any,
+    ) -> dict[str, Any]:
+        """
+        Run the full pipeline: find path → listdir → filter .txt/.md → token-chunk → push to Chroma.
+        No API; everything runs in the library.
+        """
+        _check_chroma_deps()
+
+        path = Path(path).resolve()
+        if not path.exists():
+            raise ValueError(f"path does not exist: {path}")
+
+        if collection_name == "hecvec":
+            collection_name = path.stem if path.is_file() else path.name
+
+        if path.is_file():
+            if path.suffix.lower() not in (".txt", ".md"):
+                raise ValueError(f"File must be .txt or .md: {path}")
+            root = path.parent
+            paths = [path]
+            logger.info("Archivo a procesar: %s", path)
+        else:
+            root = Path(root).resolve() if root else path
+            if not root.is_dir():
+                raise ValueError(f"path must be an existing directory: {root}")
+            logger.info("Ruta a procesar: %s", root)
+            lister = ListDirTextFiles(root=root)
+            paths = lister.listdir_recursive_txt_md(path)
+            if not paths:
+                logger.warning("No se encontraron archivos .txt/.md en %s", path)
+                return {"files": 0, "chunks": 0, "collection": collection_name, "message": "No .txt/.md files found"}
+            logger.info("Archivos .txt/.md encontrados: %d", len(paths))
+            for i, p in enumerate(paths[:10]):
+                logger.info("  %s", p)
+            if len(paths) > 10:
+                logger.info("  ... y %d más", len(paths) - 10)
+
+        # 1. (paths already set above) — 2. Read as text
+        logger.info("Leyendo contenido de %d archivos...", len(paths))
+        reader = ReadText(paths)
+        path_and_text = reader.read_all()
+        logger.info("Leídos %d archivos correctamente", len(path_and_text))
+
+        # 3. Token chunk
+        logger.info("Fragmentando con chunk_size=%d, chunk_overlap=%d...", chunk_size, chunk_overlap)
+        ids, documents = token_chunk_documents(
+            path_and_text,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            encoding_name=encoding_name,
+        )
+        if not documents:
+            logger.warning("No se generaron chunks (archivos vacíos o sin texto)")
+            return {"files": len(path_and_text), "chunks": 0, "collection": collection_name}
+
+        logger.info("Chunks generados: %d", len(documents))
+
+        # 4. Embed
+        api_key = openai_api_key or load_openai_key(dotenv_path)
+        if not api_key:
+            raise ValueError(
+                "OPENAI_API_KEY required for embeddings. "
+                "Set it in .env, pass openai_api_key=, or set the OPENAI_API_KEY env var."
+            )
+        logger.info("Generando embeddings (modelo=%s, batch_size=%d)...", embedding_model, batch_size)
+        embeddings = embed_texts(
+            documents,
+            api_key=api_key,
+            model=embedding_model,
+            batch_size=batch_size,
+        )
+        logger.info("Embeddings generados: %d vectores", len(embeddings))
+
+        # 5. Push to Chroma
+        logger.info("Conectando a Chroma (host=%s, port=%s)...", chroma_host, chroma_port)
+        client = get_client(host=chroma_host, port=chroma_port)
+        logger.info("Añadiendo %d documentos a la colección '%s'...", len(documents), collection_name)
+        add_documents(client, collection_name, ids, embeddings, documents)
+        logger.info("Pipeline completado: %d archivos → %d chunks → Chroma", len(path_and_text), len(documents))
+
+        return {
+            "files": len(path_and_text),
+            "chunks": len(documents),
+            "collection": collection_name,
+        }

hecvec/reading.py

@@ -0,0 +1,59 @@
+"""Read text files and expose content as strings."""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# Fallback encodings to try if UTF-8 fails (e.g. Windows Latin-1 / CP1252)
+FALLBACK_ENCODINGS = ("utf-8", "latin-1", "cp1252")
+
+
+class ReadText:
+    """
+    Given a list of file paths (.txt and .md), reads each as text.
+    Returns (path, text) pairs. Tries UTF-8 first, then latin-1, then cp1252.
+    """
+
+    def __init__(self, paths: list[str] | list[Path], encoding: str = "utf-8"):
+        self.paths = [Path(p) for p in paths]
+        self.encoding = encoding
+
+    def read_all(self) -> list[tuple[Path, str]]:
+        """
+        Read each file as text.
+        Returns a list of (path, text). Skips non-files and missing paths.
+        """
+        out = []
+        for p in self.paths:
+            if not p.is_file():
+                logger.debug("Saltando (no es archivo): %s", p)
+                continue
+            text = None
+            for enc in FALLBACK_ENCODINGS:
+                try:
+                    text = p.read_text(encoding=enc)
+                    if enc != "utf-8":
+                        logger.info("Leído con encoding %s: %s", enc, p.name)
+                    break
+                except (OSError, UnicodeDecodeError) as e:
+                    if enc == FALLBACK_ENCODINGS[-1]:
+                        logger.warning("No se pudo leer %s: %s", p, e)
+                    continue
+            if text is not None:
+                out.append((p.resolve(), text))
+        return out
+
+    def __iter__(self):
+        """Iterate over (path, text)."""
+        for p in self.paths:
+            if not p.is_file():
+                continue
+            for enc in FALLBACK_ENCODINGS:
+                try:
+                    text = p.read_text(encoding=enc)
+                    yield p.resolve(), text
+                    break
+                except (OSError, UnicodeDecodeError):
+                    continue

hecvec/token_splitter.py

@@ -0,0 +1,67 @@
+"""
+Token-based text chunking (TokenTextSplitter from langchain).
+One module = one responsibility: split text by token count.
+Requires: pip install hecvec[chunk] or hecvec[chroma]
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from langchain_text_splitters import TokenTextSplitter
+
+DEFAULT_CHUNK_SIZE = 200
+DEFAULT_CHUNK_OVERLAP = 0
+DEFAULT_ENCODING_NAME = "cl100k_base"
+
+
+def _get_token_splitter(
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
+    encoding_name: str = DEFAULT_ENCODING_NAME,
+) -> "TokenTextSplitter":
+    try:
+        from langchain_text_splitters import TokenTextSplitter
+    except ImportError as e:
+        raise ImportError(
+            "Token splitting requires langchain-text-splitters. Install with: pip install hecvec[chunk]"
+        ) from e
+    return TokenTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        encoding_name=encoding_name,
+    )
+
+
+def token_chunk_text(
+    text: str,
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
+    encoding_name: str = DEFAULT_ENCODING_NAME,
+) -> list[str]:
+    """Split a single document text into chunks by token count."""
+    splitter = _get_token_splitter(chunk_size, chunk_overlap, encoding_name)
+    return splitter.split_text(text)
+
+
+def token_chunk_documents(
+    path_and_texts: list[tuple[str | Path, str]],
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
+    encoding_name: str = DEFAULT_ENCODING_NAME,
+) -> tuple[list[str], list[str]]:
+    """
+    Chunk multiple documents by token count.
+    Returns (ids, documents) where ids are chunk_0, chunk_1, ...
+    """
+    splitter = _get_token_splitter(chunk_size, chunk_overlap, encoding_name)
+    documents: list[str] = []
+    ids: list[str] = []
+    for path, text in path_and_texts:
+        for content in splitter.split_text(text):
+            if not content.strip():
+                continue
+            ids.append(f"chunk_{len(ids)}")
+            documents.append(content)
+    return ids, documents

hecvec-0.1.0.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: hecvec
+Version: 0.1.0
+Summary: List directories (safe root), filter .txt/.md files, read as text, chunk, embed, and push to Chroma.
+License-Expression: MIT
+Keywords: chunking,document-pipeline,listdir,text-files
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <3.14,>=3.9
+Requires-Dist: chromadb>=0.4.0
+Requires-Dist: langchain-text-splitters>=0.2.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: tiktoken>=0.5.0
+Provides-Extra: chroma
+Requires-Dist: chromadb>=0.4.0; extra == 'chroma'
+Requires-Dist: langchain-text-splitters>=0.2.0; extra == 'chroma'
+Requires-Dist: openai>=1.0.0; extra == 'chroma'
+Requires-Dist: python-dotenv>=1.0.0; extra == 'chroma'
+Requires-Dist: tiktoken>=0.5.0; extra == 'chroma'
+Provides-Extra: chunk
+Requires-Dist: langchain-text-splitters>=0.2.0; extra == 'chunk'
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# HecVec
+
+List directories with a safe root, filter `.txt`/`.md` files, read them as text, and optionally chunk and push to Chroma — **library only, no API**.
+
+## Install
+
+```bash
+pip install hecvec
+```
+
+One-call pipeline (list → filter → token-chunk → Chroma):
+
+```bash
+pip install hecvec[chroma]
+```
+
+Optional chunking only (no Chroma):
+
+```bash
+pip install hecvec[chunk]
+```
+
+## Usage
+
+### One-call pipeline (list → filter → chunk → Chroma)
+
+Runs entirely in the library (no API). You need Chroma running (e.g. `docker run -p 8000:8000 chromadb/chroma`) and `OPENAI_API_KEY` set (in the environment or in a `.env` file; the library loads `.env` via python-dotenv when you use `hecvec[chroma]`).
+
+```python
+import hecvec
+
+# Class-style: use defaults, then slice
+test = hecvec.HecVec()
+result = test.slice(path="/path/to/folder")
+# → {"files": N, "chunks": M, "collection": "hecvec"}
+
+# Or call slice on the class (same flow)
+result = hecvec.HecVec.slice(path="/path/to/folder")
+```
+
+Flow: resolve path → listdir → filter `.txt`/`.md` → token-chunk (200 tokens, `cl100k_base`) → embed with OpenAI → push to Chroma.
+
+Optional config (instance or `HecVec.slice(..., key=value)`):
+
+- `root`, `collection_name`, `chroma_host`, `chroma_port`
+- `embedding_model`, `chunk_size`, `chunk_overlap`, `encoding_name`, `batch_size`
+- `openai_api_key` (or set `OPENAI_API_KEY` in the environment or in a `.env` file; optional `dotenv_path` to point to a specific `.env`)
+
+### Low-level building blocks
+
+```python
+from pathlib import Path
+from hecvec import ListDir, ListDirTextFiles, ReadText
+
+root = Path("/path/to/repo")
+
+# List all entries under a path (restricted to root)
+lister = ListDir(root=root)
+for rel in lister.listdir("."):
+    print(rel)
+
+# Only .txt and .md files, recursively
+text_lister = ListDirTextFiles(root=root)
+paths = text_lister.listdir_recursive_txt_md("docs")
+
+# Read each file as text
+reader = ReadText(paths)
+for path, text in reader:
+    print(path, len(text))
+```
+
+### Chunking (optional)
+
+With `pip install hecvec[chunk]`:
+
+```python
+from hecvec import ListDirTextFiles, ReadText
+from hecvec.chunking import chunk_documents
+
+lister = ListDirTextFiles(root=root)
+paths = lister.listdir_recursive_txt_md(".")
+reader = ReadText(paths)
+path_and_text = reader.read_all()
+chunks = chunk_documents(path_and_text)
+# list of {"path": "...", "chunk_index": 0, "content": "..."}
+```
+
+### CLI
+
+```bash
+hecvec-listdir [path] [root]
+# or
+python -m hecvec.cli [path] [root]
+```
+
+### Test the full pipeline (the method that does everything)
+
+From the project root, with Chroma running and `OPENAI_API_KEY` set (e.g. in `.env`):
+
+```bash
+# Start Chroma (one terminal)
+docker run -p 8000:8000 chromadb/chroma
+
+# Run the test script (another terminal)
+uv run python scripts/test_slice.py
+# or: python scripts/test_slice.py
+```
+
+The script creates a temp folder with two `.txt` files, runs `HecVec.slice(path=...)`, and prints `PASS` or `FAIL` with the result (`files`, `chunks`, `collection`).
+
+### Modular layout (easy to study)
+
+Each step of the pipeline lives in its own module:
+
+| Module | Responsibility |
+|--------|-----------------|
+| `hecvec.env` | Load `.env` and `OPENAI_API_KEY` |
+| `hecvec.listdir` | List dirs under a safe root; filter by extension (`.txt`/`.md`) |
+| `hecvec.reading` | Read files as text (UTF-8 / latin-1 / cp1252 fallback) |
+| `hecvec.token_splitter` | Token-based chunking (TokenTextSplitter) |
+| `hecvec.chunking` | Recursive-character chunking (RecursiveCharacterTextSplitter) |
+| `hecvec.embeddings` | OpenAI embeddings (`embed_texts`) |
+| `hecvec.chroma_client` | Chroma client, get/create collection, add documents |
+| `hecvec.chroma_list` | List Chroma collections and counts |
+| `hecvec.pipeline` | Orchestrator: `HecVec` and `slice(path=...)` |
+
+Example: use one step on its own:
+
+```python
+from hecvec import embed_texts, token_chunk_text, list_collections
+
+chunks = token_chunk_text("Some long document...", chunk_size=200)
+vecs = embed_texts(chunks, api_key="sk-...")
+names_and_counts = list_collections(host="localhost", port=8000)
+```
+
+## Development
+
+From the repo root:
+
+```bash
+uv sync
+uv run python -c "from hecvec import ListDir; print(ListDir('.').listdir('.'))"
+```
+
+## License
+
+MIT

hecvec-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+hecvec/__init__.py,sha256=mwGyDxZgshlfn1wWUB5xO2LTwg5-LWYQmD3Io6KGIZk,1028
+hecvec/chroma_client.py,sha256=iNQDw5r2_BReLY7MVhoJHhXvJduBv_0MR0UkeDEtKf0,1779
+hecvec/chroma_list.py,sha256=botRTK0Ge_-PwBsj9-WE0VxjyMJa6g07m1MO1_1pq2Q,722
+hecvec/chunking.py,sha256=BpHjL5e7I8u-T6goU0TD_Y1US63GUEz_tYIu2iTN9U0,2203
+hecvec/cli.py,sha256=oMernmxfk-_TZr83b0-jl3STl5xBRVORqTokcDByZBA,642
+hecvec/embeddings.py,sha256=kvjty_BDx77COehxTvHWXcKsNqMy2iKiT2H_7V9wp4k,1149
+hecvec/env.py,sha256=691efRFxSzEtDgtM0xl5aAbgrCvjgUqSQe7dG8jzV9g,778
+hecvec/hecvec.py,sha256=XGBXBCAZhdOzP5xvDIFnZZtsMBNOZXrt_dcrIg1vlFE,153
+hecvec/listdir.py,sha256=1xRR7Rn9hAhIscsweZtLBTq54BIV7G8yI9w_qySEIEY,4428
+hecvec/pipeline.py,sha256=TwYBG9Bc06Z04oIAi-vYtCH5u6hjXt853EwR3mJVH_s,7388
+hecvec/reading.py,sha256=gPPVuQPukJIKUJS69-BBZOwMgnvx-0da8gQJ5D3P6sM,2003
+hecvec/token_splitter.py,sha256=usKZjNw_BQasuNx9Hz2iGUHUiq8fNYIy_HMiW7QaZ1k,2167
+hecvec-0.1.0.dist-info/METADATA,sha256=Q3FZQUC_eeBUKOHvQI0CpgaqifMjpYi_ZU2m5sXl5dw,5704
+hecvec-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+hecvec-0.1.0.dist-info/entry_points.txt,sha256=M-5kL-e8yJH7OxTWqnQhLWJhuDPzBU0_rbpr7RVcLvQ,51
+hecvec-0.1.0.dist-info/RECORD,,

hecvec-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

hecvec-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hecvec-listdir = hecvec.cli:main