hecvec 0.1.0__tar.gz

hecvec-0.1.0/.devcontainer/Dockerfile

@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/devcontainers/base:noble
+
+# Instalar Python y dependencias
+RUN apt-get update && apt-get install -y \
+    python3 \
+    python3-pip \
+    python3-venv \
+    && rm -rf /var/lib/apt/lists/*
+
+# Instalar boto3, chromadb-client y uv
+RUN pip3 install --break-system-packages boto3 chromadb-client uv
+
+# Crear directorio de trabajo
+WORKDIR /workspace

hecvec-0.1.0/.devcontainer/devcontainer.json

@@ -0,0 +1,11 @@
+{
+	"name": "Ubuntu",
+	"image": "mcr.microsoft.com/devcontainers/base:noble",
+	"features": {
+		"ghcr.io/devcontainers/features/aws-cli:1": {},
+		"ghcr.io/devcontainers/features/node:1": {},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
+		"ghcr.io/va-h/devcontainers-features/uv:1": {}
+	},
+	"postCreateCommand": "uv sync --extra chroma"
+}

hecvec-0.1.0/.devcontainer/docker-compose.yml

@@ -0,0 +1,39 @@
+version: '3.8'
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    
+    volumes:
+      - ../:/workspace:cached
+      - /Users/toro/.ssh:/home/vscode/.ssh-host:ro
+      - /Users/toro/.aws:/home/vscode/.aws:cached
+    
+    command: sleep infinity
+    
+    environment:
+      AWS_PROFILE: default
+      AWS_DEFAULT_REGION: us-east-2
+    
+    depends_on:
+      - chromadb
+    
+    network_mode: service:chromadb
+
+  chromadb:
+    image: chromadb/chroma:latest
+    ports:
+      - "8000:8000"
+    
+    volumes:
+      - chroma-data:/chroma/data
+    
+    environment:
+      - IS_PERSISTENT=TRUE
+      - ANONYMIZED_TELEMETRY=FALSE
+
+volumes:
+  chroma-data:
+    driver: local

hecvec-0.1.0/.gitignore

@@ -0,0 +1,25 @@
+# Environment and secrets
+.env
+.venv/
+venv/
+env/
+
+# Build and publish
+dist/
+build/
+*.egg-info/
+*.egg
+
+# IDE and OS
+.idea/
+.vscode/
+*.swp
+.DS_Store
+
+# pytest
+.pytest_cache/
+.coverage
+htmlcov/
+
+# uv
+.uv/

hecvec-0.1.0/PKG-INFO

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: hecvec
+Version: 0.1.0
+Summary: List directories (safe root), filter .txt/.md files, read as text, chunk, embed, and push to Chroma.
+License-Expression: MIT
+Keywords: chunking,document-pipeline,listdir,text-files
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: <3.14,>=3.9
+Requires-Dist: chromadb>=0.4.0
+Requires-Dist: langchain-text-splitters>=0.2.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: tiktoken>=0.5.0
+Provides-Extra: chroma
+Requires-Dist: chromadb>=0.4.0; extra == 'chroma'
+Requires-Dist: langchain-text-splitters>=0.2.0; extra == 'chroma'
+Requires-Dist: openai>=1.0.0; extra == 'chroma'
+Requires-Dist: python-dotenv>=1.0.0; extra == 'chroma'
+Requires-Dist: tiktoken>=0.5.0; extra == 'chroma'
+Provides-Extra: chunk
+Requires-Dist: langchain-text-splitters>=0.2.0; extra == 'chunk'
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# HecVec
+
+List directories with a safe root, filter `.txt`/`.md` files, read them as text, and optionally chunk and push to Chroma — **library only, no API**.
+
+## Install
+
+```bash
+pip install hecvec
+```
+
+One-call pipeline (list → filter → token-chunk → Chroma):
+
+```bash
+pip install hecvec[chroma]
+```
+
+Optional chunking only (no Chroma):
+
+```bash
+pip install hecvec[chunk]
+```
+
+## Usage
+
+### One-call pipeline (list → filter → chunk → Chroma)
+
+Runs entirely in the library (no API). You need Chroma running (e.g. `docker run -p 8000:8000 chromadb/chroma`) and `OPENAI_API_KEY` set (in the environment or in a `.env` file; the library loads `.env` via python-dotenv when you use `hecvec[chroma]`).
+
+```python
+import hecvec
+
+# Class-style: use defaults, then slice
+test = hecvec.HecVec()
+result = test.slice(path="/path/to/folder")
+# → {"files": N, "chunks": M, "collection": "hecvec"}
+
+# Or call slice on the class (same flow)
+result = hecvec.HecVec.slice(path="/path/to/folder")
+```
+
+Flow: resolve path → listdir → filter `.txt`/`.md` → token-chunk (200 tokens, `cl100k_base`) → embed with OpenAI → push to Chroma.
+
+Optional config (instance or `HecVec.slice(..., key=value)`):
+
+- `root`, `collection_name`, `chroma_host`, `chroma_port`
+- `embedding_model`, `chunk_size`, `chunk_overlap`, `encoding_name`, `batch_size`
+- `openai_api_key` (or set `OPENAI_API_KEY` in the environment or in a `.env` file; optional `dotenv_path` to point to a specific `.env`)
+
+### Low-level building blocks
+
+```python
+from pathlib import Path
+from hecvec import ListDir, ListDirTextFiles, ReadText
+
+root = Path("/path/to/repo")
+
+# List all entries under a path (restricted to root)
+lister = ListDir(root=root)
+for rel in lister.listdir("."):
+    print(rel)
+
+# Only .txt and .md files, recursively
+text_lister = ListDirTextFiles(root=root)
+paths = text_lister.listdir_recursive_txt_md("docs")
+
+# Read each file as text
+reader = ReadText(paths)
+for path, text in reader:
+    print(path, len(text))
+```
+
+### Chunking (optional)
+
+With `pip install hecvec[chunk]`:
+
+```python
+from hecvec import ListDirTextFiles, ReadText
+from hecvec.chunking import chunk_documents
+
+lister = ListDirTextFiles(root=root)
+paths = lister.listdir_recursive_txt_md(".")
+reader = ReadText(paths)
+path_and_text = reader.read_all()
+chunks = chunk_documents(path_and_text)
+# list of {"path": "...", "chunk_index": 0, "content": "..."}
+```
+
+### CLI
+
+```bash
+hecvec-listdir [path] [root]
+# or
+python -m hecvec.cli [path] [root]
+```
+
+### Test the full pipeline (the method that does everything)
+
+From the project root, with Chroma running and `OPENAI_API_KEY` set (e.g. in `.env`):
+
+```bash
+# Start Chroma (one terminal)
+docker run -p 8000:8000 chromadb/chroma
+
+# Run the test script (another terminal)
+uv run python scripts/test_slice.py
+# or: python scripts/test_slice.py
+```
+
+The script creates a temp folder with two `.txt` files, runs `HecVec.slice(path=...)`, and prints `PASS` or `FAIL` with the result (`files`, `chunks`, `collection`).
+
+### Modular layout (easy to study)
+
+Each step of the pipeline lives in its own module:
+
+| Module | Responsibility |
+|--------|-----------------|
+| `hecvec.env` | Load `.env` and `OPENAI_API_KEY` |
+| `hecvec.listdir` | List dirs under a safe root; filter by extension (`.txt`/`.md`) |
+| `hecvec.reading` | Read files as text (UTF-8 / latin-1 / cp1252 fallback) |
+| `hecvec.token_splitter` | Token-based chunking (TokenTextSplitter) |
+| `hecvec.chunking` | Recursive-character chunking (RecursiveCharacterTextSplitter) |
+| `hecvec.embeddings` | OpenAI embeddings (`embed_texts`) |
+| `hecvec.chroma_client` | Chroma client, get/create collection, add documents |
+| `hecvec.chroma_list` | List Chroma collections and counts |
+| `hecvec.pipeline` | Orchestrator: `HecVec` and `slice(path=...)` |
+
+Example: use one step on its own:
+
+```python
+from hecvec import embed_texts, token_chunk_text, list_collections
+
+chunks = token_chunk_text("Some long document...", chunk_size=200)
+vecs = embed_texts(chunks, api_key="sk-...")
+names_and_counts = list_collections(host="localhost", port=8000)
+```
+
+## Development
+
+From the repo root:
+
+```bash
+uv sync
+uv run python -c "from hecvec import ListDir; print(ListDir('.').listdir('.'))"
+```
+
+## License
+
+MIT

hecvec-0.1.0/README.md

@@ -0,0 +1,148 @@
+# HecVec
+
+List directories with a safe root, filter `.txt`/`.md` files, read them as text, and optionally chunk and push to Chroma — **library only, no API**.
+
+## Install
+
+```bash
+pip install hecvec
+```
+
+One-call pipeline (list → filter → token-chunk → Chroma):
+
+```bash
+pip install hecvec[chroma]
+```
+
+Optional chunking only (no Chroma):
+
+```bash
+pip install hecvec[chunk]
+```
+
+## Usage
+
+### One-call pipeline (list → filter → chunk → Chroma)
+
+Runs entirely in the library (no API). You need Chroma running (e.g. `docker run -p 8000:8000 chromadb/chroma`) and `OPENAI_API_KEY` set (in the environment or in a `.env` file; the library loads `.env` via python-dotenv when you use `hecvec[chroma]`).
+
+```python
+import hecvec
+
+# Class-style: use defaults, then slice
+test = hecvec.HecVec()
+result = test.slice(path="/path/to/folder")
+# → {"files": N, "chunks": M, "collection": "hecvec"}
+
+# Or call slice on the class (same flow)
+result = hecvec.HecVec.slice(path="/path/to/folder")
+```
+
+Flow: resolve path → listdir → filter `.txt`/`.md` → token-chunk (200 tokens, `cl100k_base`) → embed with OpenAI → push to Chroma.
+
+Optional config (instance or `HecVec.slice(..., key=value)`):
+
+- `root`, `collection_name`, `chroma_host`, `chroma_port`
+- `embedding_model`, `chunk_size`, `chunk_overlap`, `encoding_name`, `batch_size`
+- `openai_api_key` (or set `OPENAI_API_KEY` in the environment or in a `.env` file; optional `dotenv_path` to point to a specific `.env`)
+
+### Low-level building blocks
+
+```python
+from pathlib import Path
+from hecvec import ListDir, ListDirTextFiles, ReadText
+
+root = Path("/path/to/repo")
+
+# List all entries under a path (restricted to root)
+lister = ListDir(root=root)
+for rel in lister.listdir("."):
+    print(rel)
+
+# Only .txt and .md files, recursively
+text_lister = ListDirTextFiles(root=root)
+paths = text_lister.listdir_recursive_txt_md("docs")
+
+# Read each file as text
+reader = ReadText(paths)
+for path, text in reader:
+    print(path, len(text))
+```
+
+### Chunking (optional)
+
+With `pip install hecvec[chunk]`:
+
+```python
+from hecvec import ListDirTextFiles, ReadText
+from hecvec.chunking import chunk_documents
+
+lister = ListDirTextFiles(root=root)
+paths = lister.listdir_recursive_txt_md(".")
+reader = ReadText(paths)
+path_and_text = reader.read_all()
+chunks = chunk_documents(path_and_text)
+# list of {"path": "...", "chunk_index": 0, "content": "..."}
+```
+
+### CLI
+
+```bash
+hecvec-listdir [path] [root]
+# or
+python -m hecvec.cli [path] [root]
+```
+
+### Test the full pipeline (the method that does everything)
+
+From the project root, with Chroma running and `OPENAI_API_KEY` set (e.g. in `.env`):
+
+```bash
+# Start Chroma (one terminal)
+docker run -p 8000:8000 chromadb/chroma
+
+# Run the test script (another terminal)
+uv run python scripts/test_slice.py
+# or: python scripts/test_slice.py
+```
+
+The script creates a temp folder with two `.txt` files, runs `HecVec.slice(path=...)`, and prints `PASS` or `FAIL` with the result (`files`, `chunks`, `collection`).
+
+### Modular layout (easy to study)
+
+Each step of the pipeline lives in its own module:
+
+| Module | Responsibility |
+|--------|-----------------|
+| `hecvec.env` | Load `.env` and `OPENAI_API_KEY` |
+| `hecvec.listdir` | List dirs under a safe root; filter by extension (`.txt`/`.md`) |
+| `hecvec.reading` | Read files as text (UTF-8 / latin-1 / cp1252 fallback) |
+| `hecvec.token_splitter` | Token-based chunking (TokenTextSplitter) |
+| `hecvec.chunking` | Recursive-character chunking (RecursiveCharacterTextSplitter) |
+| `hecvec.embeddings` | OpenAI embeddings (`embed_texts`) |
+| `hecvec.chroma_client` | Chroma client, get/create collection, add documents |
+| `hecvec.chroma_list` | List Chroma collections and counts |
+| `hecvec.pipeline` | Orchestrator: `HecVec` and `slice(path=...)` |
+
+Example: use one step on its own:
+
+```python
+from hecvec import embed_texts, token_chunk_text, list_collections
+
+chunks = token_chunk_text("Some long document...", chunk_size=200)
+vecs = embed_texts(chunks, api_key="sk-...")
+names_and_counts = list_collections(host="localhost", port=8000)
+```
+
+## Development
+
+From the repo root:
+
+```bash
+uv sync
+uv run python -c "from hecvec import ListDir; print(ListDir('.').listdir('.'))"
+```
+
+## License
+
+MIT

hecvec-0.1.0/pyproject.toml

@@ -0,0 +1,53 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "hecvec"
+version = "0.1.0"
+description = "List directories (safe root), filter .txt/.md files, read as text, chunk, embed, and push to Chroma."
+readme = "README.md"
+requires-python = ">=3.9,<3.14"
+license = "MIT"
+authors = []
+keywords = ["listdir", "text-files", "chunking", "document-pipeline"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "chromadb>=0.4.0",
+    "langchain-text-splitters>=0.2.0",
+    "openai>=1.0.0",
+    "python-dotenv>=1.0.0",
+    "tiktoken>=0.5.0",
+]
+
+[project.optional-dependencies]
+chunk = ["langchain-text-splitters>=0.2.0"]
+chroma = [
+    "chromadb>=0.4.0",
+    "langchain-text-splitters>=0.2.0",
+    "openai>=1.0.0",
+    "python-dotenv>=1.0.0",
+    "tiktoken>=0.5.0",
+]
+dev = ["pytest>=7.0.0"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+
+[project.scripts]
+hecvec-listdir = "hecvec.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/hecvec"]

hecvec-0.1.0/scripts/test_slice.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""
+Create a HecVec object and test the method that does everything by passing your path.
+
+  listdir → filter .txt/.md → token-chunk → embed → Chroma
+
+Requirements:
+  - pip install hecvec[chroma]
+  - Chroma running: docker run -p 8000:8000 chromadb/chroma
+  - OPENAI_API_KEY set (env or .env in project root)
+
+Usage (from project root):
+  uv run python scripts/test_slice.py /path/to/folder
+  uv run python scripts/test_slice.py .
+  python scripts/test_slice.py /path/to/folder
+"""
+import logging
+import sys
+from pathlib import Path
+
+# Project root and src
+ROOT = Path(__file__).resolve().parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+# Default path to slice: project root (works in devcontainer; use first arg to override)
+DEFAULT_PATH = ROOT
+
+
+def main() -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+        datefmt="%H:%M:%S",
+    )
+    logger = logging.getLogger(__name__)
+
+    from hecvec import HecVec
+
+    if len(sys.argv) < 2:
+        path = DEFAULT_PATH
+        logger.info("No path given, using default: %s", path)
+    else:
+        path = Path(sys.argv[1]).expanduser().resolve()
+        logger.info("Path given: %s", path)
+
+    if not path.exists():
+        logger.error("Path does not exist: %s", path)
+        print(f"Error: path does not exist: {path}")
+        return 1
+    if not path.is_dir() and not path.is_file():
+        logger.error("Not a file or directory: %s", path)
+        print(f"Error: not a file or directory: {path}")
+        return 1
+    if path.is_file() and path.suffix.lower() not in (".txt", ".md"):
+        logger.error("File must be .txt or .md: %s", path)
+        print(f"Error: file must be .txt or .md: {path}")
+        return 1
+
+    test = HecVec()
+    result = test.slice(path=path)
+
+    logger.info("Result: %s", result)
+    print("Result:", result)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

hecvec-0.1.0/src/hecvec/__init__.py

@@ -0,0 +1,34 @@
+"""
+HecVec: modular library — listdir, read, chunk (token/recursive), embed, Chroma. No API.
+"""
+from hecvec.chunking import chunk_documents, chunk_text
+from hecvec.chroma_client import add_documents, get_client, get_or_create_collection
+from hecvec.chroma_list import list_collections
+from hecvec.embeddings import embed_texts
+from hecvec.env import load_dotenv_if_available, load_openai_key
+from hecvec.listdir import ALLOWED_EXTENSIONS, ListDir, ListDirTextFiles
+from hecvec.pipeline import HecVec
+from hecvec.reading import ReadText
+from hecvec.token_splitter import token_chunk_documents, token_chunk_text
+
+__all__ = [
+    "ALLOWED_EXTENSIONS",
+    "HecVec",
+    "ListDir",
+    "ListDirTextFiles",
+    "ReadText",
+    "add_documents",
+    "chunk_documents",
+    "chunk_text",
+    "embed_texts",
+    "get_client",
+    "get_or_create_collection",
+    "list_collections",
+    "load_dotenv_if_available",
+    "load_openai_key",
+    "token_chunk_documents",
+    "token_chunk_text",
+    "__version__",
+]
+
+__version__ = "0.1.0"

hecvec-0.1.0/src/hecvec/chroma_client.py

@@ -0,0 +1,58 @@
+"""
+Chroma client and collection operations. One module = one responsibility: connect and add documents.
+Requires: pip install hecvec[chroma] (chromadb).
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import chromadb
+
+DEFAULT_HOST = "localhost"
+DEFAULT_PORT = 8000
+
+
+def get_client(host: str = DEFAULT_HOST, port: int = DEFAULT_PORT):
+    """Return a Chroma HTTP client, or an in-memory client if the server is not reachable."""
+    import chromadb
+    try:
+        return chromadb.HttpClient(host=host, port=port)
+    except Exception:
+        return chromadb.EphemeralClient()
+
+
+def get_or_create_collection(
+    client: "chromadb.HttpClient",
+    name: str,
+    metadata: dict | None = None,
+):
+    """Get or create a collection (cosine similarity by default)."""
+    if metadata is None:
+        metadata = {"hnsw:space": "cosine"}
+    return client.get_or_create_collection(name=name, metadata=metadata)
+
+
+def add_documents(
+    client: "chromadb.HttpClient",
+    collection_name: str,
+    ids: list[str],
+    embeddings: list[list[float]],
+    documents: list[str],
+) -> None:
+    """
+    Add documents to a collection. If dimension mismatch, deletes and recreates the collection.
+    """
+    import chromadb
+    coll = get_or_create_collection(client, collection_name)
+    try:
+        coll.add(ids=ids, embeddings=embeddings, documents=documents)
+    except chromadb.errors.InvalidArgumentError as e:
+        if "dimension" not in str(e).lower():
+            raise
+        client.delete_collection(name=collection_name)
+        coll = client.create_collection(
+            name=collection_name,
+            metadata={"hnsw:space": "cosine"},
+        )
+        coll.add(ids=ids, embeddings=embeddings, documents=documents)

hecvec-0.1.0/src/hecvec/chroma_list.py

@@ -0,0 +1,27 @@
+"""
+List Chroma collections. One module = one responsibility: inspect collections on a Chroma server.
+Requires: pip install hecvec[chroma] (chromadb).
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import chromadb
+
+DEFAULT_HOST = "localhost"
+DEFAULT_PORT = 8000
+
+
+def list_collections(
+    host: str = DEFAULT_HOST,
+    port: int = DEFAULT_PORT,
+) -> list[tuple[str, int]]:
+    """
+    List all collection names and their document counts on a Chroma server.
+    Returns [(name, count), ...].
+    """
+    import chromadb
+    client = chromadb.HttpClient(host=host, port=port)
+    collections = client.list_collections()
+    return [(c.name, c.count()) for c in collections]