chunky-files 0.2.0__tar.gz

chunky_files-0.2.0/.gitignore

@@ -0,0 +1,47 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
+.eggs/
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+pytest_cache/
+.coverage
+coverage.xml
+
+# Sphinx build artifacts
+docs/_build/
+
+# IDEs and editors
+.vscode/
+.idea/
+*.swp
+
+# macOS
+.DS_Store
+
+# Hatch environments
+.hatch/
+
+# Environment file
+.env
+.venv/
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

chunky_files-0.2.0/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.2.0] - TBD
+### Added
+- Changelog (`CHANGELOG.md`; this file).
+- Release process section added to the existing `README.md`
+- `PYPI_TOKEN`, `TEST_PYPI_TOKEN`, and `CODECOV_TOKEN` added to github secrets
+- `.env` and other common evironment file name added to the `.gitignore` for token security. 
+### Changes
+- Release workflow updated to have matching secrets name.
+### Fixes
+- Updated dependencies and improve type hints in codebase (ruff compliance).
+- Update build tooling installation in release .
+- Included pyproject.toml in sdist build targets.
+
+## [0.1.0] - 2025-09-30
+### Added
+- Initial project scaffolding with Hatchling build system and CI/release workflows.
+- Core chunking data models (`Document`, `Chunk`, `ChunkerConfig`).
+- Sliding-window fallback chunker with metadata-rich outputs.
+- `ChunkPipeline` orchestration, registry, and filesystem loader.
+- Sphinx documentation skeleton and Read the Docs configuration.
+- Pytest and Ruff tooling with baseline tests for the sliding-window chunker.

chunky_files-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Nancy Brain Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chunky_files-0.2.0/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: chunky-files
+Version: 0.2.0
+Summary: Semantic chunking utilities for scientific code and documentation corpora.
+Project-URL: Home, https://github.com/AmberLee2427/chunky
+Project-URL: Documentation, https://chunky.readthedocs.io/
+Project-URL: Issues, https://github.com/AmberLee2427/chunky/issues
+Author: Nancy Brain Contributors
+License: MIT License
+        
+        Copyright (c) 2024 Nancy Brain Contributors
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.8
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: bump-my-version>=0.6; extra == 'dev'
+Requires-Dist: coverage[toml]>=7; extra == 'dev'
+Requires-Dist: pytest-cov>=4; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: furo>=2024.0.0; extra == 'docs'
+Requires-Dist: myst-parser>=2; extra == 'docs'
+Requires-Dist: sphinx>=7; extra == 'docs'
+Description-Content-Type: text/markdown
+
+# Chunky
+
+Chunky is a python package for intelligently chunking scientific and technical repositories.
+It provides a modular pipeline that powers the Nancy Brain knowledge base and MCP services,
+while remaining useful as a standalone library for retrieval systems that need deterministic,
+metadata-rich chunks.
+
+Documentation lives on Read the Docs: <https://chunky.readthedocs.io>
+
+## Installation
+
+Install from source using the `pyproject.toml` metadata:
+
+```bash
+# clone the repo (if you haven't already)
+git clone https://github.com/AmberLee2427/chunky.git
+cd chunky
+
+# install the library
+pip install .
+```
+
+For development and documentation builds, install the optional extras:
+
+```bash
+pip install -e ".[dev,docs]"
+```
+
+> `-e` performs an editable install so local changes reflect immediately.
+> `.[dev,docs]` installs the tooling declared under the `dev` and `docs` extras in
+> `pyproject.toml`.
+
+## Tooling
+
+* **Code style:** Ruff (`ruff check src tests` or `ruff check src tests --fix`)
+* **Tests:** Pytest (`pytest --cov=chunky`)
+* **Docs:** Sphinx + MyST + Furo (`sphinx-build -b html docs docs/_build/html`)
+* **Packaging:** Hatchling build backend
+* **Versioning:** bump-my-version (driven by tags and the release workflow)
+
+## Workflows
+
+* CI tests run on Linux, macOS, and Windows for Python 3.8 through 3.12.
+* Pushing a tag that matches the form `vX.Y.Z` triggers the release workflow. It validates that the
+  tag matches the version in `pyproject.toml`, builds the distribution, and publishes to PyPI using
+  the `PYPI_API_TOKEN` secret.
+* Read the Docs builds the documentation automatically for pushes to the default branch. Local
+  builds use `sphinx-build -b html docs docs/_build/html`.
+
+Release checklist:
+
+1. Review and update `CHANGELOG.md`, keeping the `[Unreleased]` section accurate.
+2. Run `bump-my-version bump <part>` to update version metadata and append a dated entry in the
+   changelog.
+3. Commit the changes and push to `main`.
+4. Tag the commit (`git tag vX.Y.Z && git push origin vX.Y.Z`) to trigger the Release workflow.
+5. Verify the PyPI publish job and Read the Docs build succeed.
+
+## Contributing
+
+* Know your audience: most contributors will be scientific coders. Write docs assuming limited
+  familiarity with packaging internals.
+* Use Ruff for style checks and keep numpy-style docstrings on all non-test functions.
+* Target test coverage above 70% and ensure existing CI jobs pass before opening a PR.
+* In pull requests, summarise code changes, testing/validation, doc updates, and provide a brief
+  TL;DR when the description runs long.
+
+## License
+
+Chunky is released under the [MIT License](LICENSE).
+
+## Glossary
+
+| Term | Meaning |
+| ---- | ------- |
+| PR | GitHub pull request – a request to merge one branch or fork with another |
+| Release | Publishing a tagged version of the project to PyPI |
+| ChangeLog | A document describing changes between releases |
+| PyPI | Python Package Index – where published distributions live |
+| Ruff | A fast Python linter/formatter used for style enforcement |
+| origin | The upstream GitHub repository |
+| fork | A downstream copy of the origin repo used for contributing |
+| master/main | The default branch |
+| CI | Continuous Integration – automated checks that run on every push/PR |
+| GitHub Workflows | GitHub’s automation runner configured via YAML files |
+| `pyproject.toml` | Core metadata and build configuration for the package |
+| bump-my-version | CLI used to bump version numbers consistently |
+| Read the Docs | Hosted documentation service that builds from the repo |

chunky_files-0.2.0/README.md

@@ -0,0 +1,88 @@
+# Chunky
+
+Chunky is a python package for intelligently chunking scientific and technical repositories.
+It provides a modular pipeline that powers the Nancy Brain knowledge base and MCP services,
+while remaining useful as a standalone library for retrieval systems that need deterministic,
+metadata-rich chunks.
+
+Documentation lives on Read the Docs: <https://chunky.readthedocs.io>
+
+## Installation
+
+Install from source using the `pyproject.toml` metadata:
+
+```bash
+# clone the repo (if you haven't already)
+git clone https://github.com/AmberLee2427/chunky.git
+cd chunky
+
+# install the library
+pip install .
+```
+
+For development and documentation builds, install the optional extras:
+
+```bash
+pip install -e ".[dev,docs]"
+```
+
+> `-e` performs an editable install so local changes reflect immediately.
+> `.[dev,docs]` installs the tooling declared under the `dev` and `docs` extras in
+> `pyproject.toml`.
+
+## Tooling
+
+* **Code style:** Ruff (`ruff check src tests` or `ruff check src tests --fix`)
+* **Tests:** Pytest (`pytest --cov=chunky`)
+* **Docs:** Sphinx + MyST + Furo (`sphinx-build -b html docs docs/_build/html`)
+* **Packaging:** Hatchling build backend
+* **Versioning:** bump-my-version (driven by tags and the release workflow)
+
+## Workflows
+
+* CI tests run on Linux, macOS, and Windows for Python 3.8 through 3.12.
+* Pushing a tag that matches the form `vX.Y.Z` triggers the release workflow. It validates that the
+  tag matches the version in `pyproject.toml`, builds the distribution, and publishes to PyPI using
+  the `PYPI_API_TOKEN` secret.
+* Read the Docs builds the documentation automatically for pushes to the default branch. Local
+  builds use `sphinx-build -b html docs docs/_build/html`.
+
+Release checklist:
+
+1. Review and update `CHANGELOG.md`, keeping the `[Unreleased]` section accurate.
+2. Run `bump-my-version bump <part>` to update version metadata and append a dated entry in the
+   changelog.
+3. Commit the changes and push to `main`.
+4. Tag the commit (`git tag vX.Y.Z && git push origin vX.Y.Z`) to trigger the Release workflow.
+5. Verify the PyPI publish job and Read the Docs build succeed.
+
+## Contributing
+
+* Know your audience: most contributors will be scientific coders. Write docs assuming limited
+  familiarity with packaging internals.
+* Use Ruff for style checks and keep numpy-style docstrings on all non-test functions.
+* Target test coverage above 70% and ensure existing CI jobs pass before opening a PR.
+* In pull requests, summarise code changes, testing/validation, doc updates, and provide a brief
+  TL;DR when the description runs long.
+
+## License
+
+Chunky is released under the [MIT License](LICENSE).
+
+## Glossary
+
+| Term | Meaning |
+| ---- | ------- |
+| PR | GitHub pull request – a request to merge one branch or fork with another |
+| Release | Publishing a tagged version of the project to PyPI |
+| ChangeLog | A document describing changes between releases |
+| PyPI | Python Package Index – where published distributions live |
+| Ruff | A fast Python linter/formatter used for style enforcement |
+| origin | The upstream GitHub repository |
+| fork | A downstream copy of the origin repo used for contributing |
+| master/main | The default branch |
+| CI | Continuous Integration – automated checks that run on every push/PR |
+| GitHub Workflows | GitHub’s automation runner configured via YAML files |
+| `pyproject.toml` | Core metadata and build configuration for the package |
+| bump-my-version | CLI used to bump version numbers consistently |
+| Read the Docs | Hosted documentation service that builds from the repo |

chunky_files-0.2.0/docs/api.rst

@@ -0,0 +1,9 @@
+API Reference
+=============
+
+.. autosummary::
+   :toctree: _autosummary
+   :caption: Public API
+   :recursive:
+
+   chunky

chunky_files-0.2.0/docs/conf.py

@@ -0,0 +1,50 @@
+"""Sphinx configuration for chunky documentation."""
+
+from __future__ import annotations
+
+import importlib.metadata
+import os
+import sys
+from datetime import datetime
+
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+SRC_DIR = os.path.join(PROJECT_ROOT, "src")
+if SRC_DIR not in sys.path:
+    sys.path.insert(0, SRC_DIR)
+
+project = "chunky"
+copyright = f"{datetime.now():%Y}, Nancy Brain Contributors"
+
+try:
+    release = importlib.metadata.version("chunky")
+except importlib.metadata.PackageNotFoundError:
+    from chunky.__about__ import __version__ as release  # type: ignore[assignment]
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.intersphinx",
+    "myst_parser",
+]
+
+autosummary_generate = True
+napoleon_google_docstring = False
+napoleon_use_param = True
+napoleon_use_rtype = True
+
+html_theme = os.environ.get("SPHINX_HTML_THEME", "furo")
+html_static_path = ["_static"]
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", {}),
+}
+
+myst_enable_extensions = [
+    "colon_fence",
+    "deflist",
+]
+
+templates_path = ["_templates"]
+
+exclude_patterns: list[str] = ["_build", "Thumbs.db", ".DS_Store"]

chunky_files-0.2.0/docs/design/GOOGLE_RESULTS.md

@@ -0,0 +1,116 @@
+## Semantic code chunking
+
+Semantic code file chunking in Python involves splitting a code file into meaningful, self-contained units based on its structure and semantics, rather than just arbitrary character counts. This approach aims to create chunks that represent logical components like functions, classes, or distinct blocks of code, improving the effectiveness of operations like embedding for RAG pipelines or code analysis. 
+
+Here's how you can achieve this in Python: 
+
+1. **Using Language-Specific Text Splitters:** 
+    Libraries like LangChain offer specialized text splitters for different programming languages. 
+    ```python
+    from langchain_experimental.text_splitter import PythonCodeTextSplitter  
+    
+    # Initialize the splitter
+    python_splitter = PythonCodeTextSplitter()
+
+    # Split the code
+    code_chunks = python_splitter.split_text(your_python_code_string)
+    ```  
+    This `PythonCodeTextSplitter` is designed to understand Python syntax and split based on elements like function definitions, class definitions, and other structural components. Similar splitters exist for other languages.  
+
+2. **Utilizing Tree-Sitter for AST-based Chunking:**
+    Tree-sitter is a parsing library that can generate Abstract Syntax Trees (ASTs) for various programming languages. You can leverage this to identify semantic boundaries more precisely. 
+
+    ```python
+    # Example using a conceptual tree-sitter based approach
+    # (Requires a tree-sitter parser for Python)
+    from tree_sitter import Language, Parser
+
+    # Load the Python language parser (you'd need to compile it first)
+    Language.build_library('build/my-languages.so', ['path/to/tree-sitter-python'])
+    PYTHON_LANGUAGE = Language('build/my-languages.so', 'python')
+
+    parser = Parser()
+    parser.set_language(PYTHON_LANGUAGE)
+
+    tree = parser.parse(bytes(your_python_code_string, 'utf8'))
+
+    # Traverse the AST to identify meaningful nodes (e.g., function definitions, class definitions)
+    # and extract their corresponding code snippets as chunks.
+    ```  
+    This method offers fine-grained control over chunking based on the exact structure of the code, but requires more manual implementation to define how AST nodes translate into chunks.  
+3. **Combining Semantic and Heuristic Approaches:** 
+    You can also combine semantic understanding with more traditional heuristic rules, such as splitting by multiple newlines or specific keywords, to create robust chunking strategies.  
+
+Considerations for Semantic Code Chunking: 
+• **Granularity:**  Decide on the appropriate level of granularity for your chunks (e.g., entire functions, individual statements, or logical blocks within functions). 
+• **Context:** Ensure that each chunk retains enough context to be meaningful on its own, especially for tasks like embedding and retrieval. 
+• **Language Specificity:** The best chunking strategy often depends on the specific programming language and its conventions. 
+
+AI responses may include mistakes.
+
+---
+
+## Semantic file chunking
+
+Semantic file chunking in Python involves splitting a document into meaningful segments based on the semantic relatedness of its content, rather than fixed-size or character-based methods. This approach aims to keep semantically coherent information together within a single chunk, which can be beneficial for tasks like Retrieval Augmented Generation (RAG) in Large Language Models (LLMs). 
+
+Here's how you can perform semantic chunking in Python:
+
+1. **Using LlamaIndex's Semantic Splitter:**  
+    LlamaIndex provides a SemanticSplitterNodeParser designed for semantic chunking.  
+    ```python
+    from llama_index.node_parser import SemanticSplitterNodeParser
+    from llama_index.embeddings import OpenAIEmbedding
+
+    # Initialize the embedding model (e.g., OpenAIEmbeddings)
+    embed_model = OpenAIEmbedding()
+
+    # Initialize the semantic splitter
+    # `buffer_size` determines how many sentences to consider for similarity comparison
+    # `breakpoint_percentile_threshold` controls the sensitivity of splitting
+    splitter = SemanticSplitterNodeParser(
+        buffer_size=1, breakpoint_percentile_threshold=95, embed_model=embed_model
+    )
+
+    # Load your document (e.g., from a file)
+    # Example: text = "Your long document text here..."
+    # Or use LlamaIndex's SimpleDirectoryReader to load documents from a directory
+
+    # Parse the document into nodes (chunks)
+    nodes = splitter.get_nodes_from_documents([document]) # Replace 'document' with your LlamaIndex Document object
+
+    # Access the content of the semantic chunks
+    for node in nodes:
+        print(node.text)
+    ```  
+2. **Using LangChain's Semantic Chunking (Experimental):** 
+   LangChain also offers an experimental SemanticChunker within langchain_experimental.  
+   ```python
+    from langchain_experimental.text_splitter import SemanticChunker
+    from langchain_openai.embeddings import OpenAIEmbeddings
+
+    # Initialize the embedding model
+    embeddings = OpenAIEmbeddings()
+
+    # Initialize the semantic chunker
+    semantic_chunker = SemanticChunker(embeddings, breakpoint_threshold_type="percentile")
+
+    # Split your text into semantic chunks
+    text = "Your long document text here..."
+    chunks = semantic_chunker.split_text(text)
+
+    for chunk in chunks:
+        print(chunk)
+    ```
+
+Key Concepts in Semantic Chunking: 
+• **Embeddings:** Text is converted into numerical vector representations (embeddings) that capture its semantic meaning. 
+• **Similarity Measurement:** The similarity between embeddings of adjacent sentences or segments is calculated (e.g., using cosine similarity). 
+• **Breakpoint Threshold:** A threshold is used to identify points where the semantic similarity drops significantly, indicating a natural break point for a new chunk. This can be based on percentiles, standard deviation, or interquartile range of similarity scores. 
+• **Adaptive Chunk Sizes:** Unlike fixed-size chunking, semantic chunking results in chunks of varying lengths, as the splits are determined by semantic coherence. 
+
+By using these methods, you can create more semantically meaningful chunks, which can lead to improved performance in downstream applications like RAG by ensuring that relevant contextual information remains together. 
+
+AI responses may include mistakes.
+
+---

chunky_files-0.2.0/docs/design/SEMANTIC_CHUNKER.md

@@ -0,0 +1,165 @@
+# Semantic Chunking Library Design
+
+## 1. Background & Motivation
+
+Our existing `SmartChunker` performs a hybrid of sliding windows and heuristic boundary searches. When processing medium-sized code files the forward/backward scans can explode in CPU and memory, causing build failures. We also have no semantic awareness for other file types, leading to arbitrary splits. To make the knowledge-base pipeline reliable and extensible we want a modular chunking library that plugs cleanly into the Nancy Brain build as well as the new MCP-based RAG service powering the Slack bot. The same library will ship as a standalone package (working name `chunky`) so other indexing services can reuse it. We want that library to:
+
+- Handles our common file types (Python, Markdown, JSON/YAML, plain text) with sensible defaults.
+- Lets us plug in stronger semantic strategies (AST, embeddings) as optional enhancements.
+- Keeps configuration centralized and easy to override via environment variables or config files.
+- Produces consistent `Chunk` objects that slot directly into the indexing pipeline.
+
+## 2. Goals & Non-Goals
+
+### Goals
+- Deterministic chunking for code and docs without pathological loops.
+- Environment-driven configuration (e.g., tweak window sizes per build).
+- Pipeline orchestration that picks the right chunker based on file metadata.
+- Clear surface for future semantic/AST-based chunkers.
+
+### Non-Goals
+- Building a full AST parser for every language on day one.
+- Re-implementing vector stores or summarization; we only prepare text for indexing/summarizing.
+- Handling binary formats such as PDFs (they stay outside this module).
+
+## 3. High-Level Architecture
+
+```
+chunky/
+├── types.py           # Chunk, Document, ChunkerConfig definitions
+├── core.py            # Chunker protocol, ChunkingError
+├── chunkers/
+│   ├── python.py      # PythonSemanticChunker (AST-aware)
+│   ├── markdown.py    # MarkdownHeadingChunker
+│   ├── yaml_json.py   # JSONYamlChunker
+│   ├── text.py        # PlainTextChunker
+│   └── fallback.py    # SlidingWindowChunker
+├── registry.py        # ChunkerRegistry + DEFAULT_REGISTRY
+├── loaders.py         # DocumentLoader hierarchy
+├── pipeline.py        # ChunkPipeline orchestrator
+└── utils.py           # Shared helpers (token counting, environment hooks)
+```
+
+The code will live in the dedicated `chunky` package and be imported by Nancy Brain (and future MCP clients) like any other dependency. Keeping the chunker in its own package keeps agent scopes narrow and makes reuse easier.
+
+## 4. Core Concepts
+
+- **Document**: normalized representation of a file with (path, content, metadata, language).
+- **Chunk**: dataclass with `chunk_id`, `text`, `metadata` (JSON-serializable), `source_document`.
+- **Chunker**: object exposing `chunk(document, config) -> List[Chunk]`.
+- **ChunkerRegistry**: resolves the appropriate chunker for a document (by extension, language, or explicit override).
+- **ChunkPipeline**: orchestrates loading, chunking, and optional summarization hooks.
+
+## 5. Chunker Implementations
+
+Minimum viable set:
+
+| Chunker | Description | Notes |
+|---------|-------------|-------|
+| `SlidingWindowChunker` | Simple fixed-line window with overlap | Always available; zero dependencies |
+| `PythonSemanticChunker` | AST-based splitting on top-level functions/classes; falls back to window | Requires `ast` (built-in). Optionally `tree_sitter` later |
+| `MarkdownHeadingChunker` | Breaks on heading hierarchy; merges small sections | No heavy deps |
+| `JSONYamlChunker` | Treats top-level keys/arrays as chunks; flatten nested objects | Uses `json` / `yaml` |
+| `PlainTextChunker` | Sentence/paragraph segmentation using regex or spaCy optional | Configurable sentence splitter |
+| `SemanticEmbeddingChunker` (optional) | Embedding-based breakpoints (cosine drift) | Depends on configured embedding model; opt-in |
+| `NotebookChunker` (via `nb4llm`) | Works with notebook-derived fenced text | Delegates heavy lifting to `nb4llm`; enforces Markdown/Python fence boundaries |
+
+Each chunker adheres to the `Chunker` protocol and accepts a `ChunkerConfig`. The fallback chunker is always used last to guarantee progress.
+
+## 6. Configuration Strategy
+
+- `ChunkerConfig` stores generic knobs (`max_chars`, `max_tokens`, `code_window_lines`, `code_overlap_lines`, `semantic_model`, etc.).
+- Defaults come from environment variables (`SMART_CHUNK_CODE_LINES`, `SMART_CHUNK_CODE_OVERLAP`, `SMART_CHUNK_TEXT_CHARS`, `SEMANTIC_MODEL`) or a YAML file (`semantic_chunker.yaml`). For MCP deployments we also respect `MCP_CHUNKER_CONFIG`, pointing to a remote-friendly YAML/JSON config path.
+- The pipeline allows per-call overrides, e.g., `pipeline.chunk_file(path, config=ChunkerConfig(code_window_lines=60))`.
+- All chunkers attach useful metadata (`line_start`, `line_end`, `language`, optional `semantic_score`) so MCP clients and Nancy's Slack responses can surface precise citations.
+
+## 7. API Sketch
+
+```python
+from chunky.pipeline import ChunkPipeline
+from chunky.types import ChunkerConfig
+
+pipeline = ChunkPipeline()  # uses DEFAULT_REGISTRY
+
+chunks = pipeline.chunk_file(
+    path="knowledge_base/raw/general_tools/Dazzle/dazzle/dazzle.py",
+    config=ChunkerConfig(
+        code_window_lines=80,
+        code_overlap_lines=10,
+        semantic_model=None,  # disable embedding-based splits
+    ),
+)
+
+for chunk in chunks:
+    print(chunk.chunk_id, chunk.metadata["line_start"], chunk.metadata["line_end"])
+```
+
+Pipeline steps internally:
+1. Load `Document` via registered loader.
+2. Resolve chunker from registry (falls back to sliding window).
+3. Invoke chunker with provided config.
+4. Optionally post-process (e.g., summarization hook, metadata enrichment).
+5. Return list of `Chunk` objects ready for indexing. Downstream consumers (Nancy Brain builders, MCP adapters, Slack bot citation tooling) rely on `chunk_id`, `source_document`, and line metadata to link answers back to source material.
+
+## 8. Integration with KB Build
+
+- Replace direct `SmartChunker` usage in `scripts/build_knowledge_base.py` with `ChunkPipeline`.
+- All metadata stays JSON-serializable; pipeline returns chunks with ready metadata.
+- Existing environment flags (`SKIP_PDF_PROCESSING`, `NB_PER_FILE_LOG`) remain untouched.
+
+## 9. Extensibility Hooks
+
+- `ChunkerRegistry.register(ext, chunker_cls)` to add new chunkers (e.g., notebook support).
+- `ChunkPipeline` accepts custom registry or pre/post hooks (e.g., run summarizer on each chunk).
+- Optional plugin entry points for projects to register chunkers via setuptools entry points.
+
+## 10. Risks & Mitigations
+
+| Risk | Mitigation |
+|------|------------|
+| AST parser errors on malformed code | Catch exceptions, fall back to sliding window |
+| Semantic chunker slows builds | Make embedding-driven chunker opt-in; default to cheapest strategy |
+| Dependency bloat | Fuck it. the heavy hitters already live in Nancy Brain and people can nuke the env after a build. I’ll plan assuming we can pull in whatever libraries make the chunkers accurate and fast; if anything starts to feel gratuitous later, we can revisit. |
+| Inconsistent metadata | Centralize metadata construction utilities; reuse JSON serialization helpers |
+
+## 11. Implementation Plan
+
+1. **Phase 1 – Infrastructure**
+   - Define `Chunk`, `Document`, `ChunkerConfig`.
+   - Implement `SlidingWindowChunker` and registry/pipeline scaffold.
+   - Swap KB build to use pipeline with sliding window (parity with current behavior).
+
+2. **Phase 2 – Language-specific chunkers**
+   - Implement `PythonSemanticChunker`, `MarkdownHeadingChunker`, `JSONYamlChunker`, `PlainTextChunker`.
+   - Add tests covering line ranges, metadata, and fallback behavior.
+
+3. **Phase 3 – Semantic chunking (optional)**
+   - Prototype embedding-based chunker using existing sentence-transformer models.
+   - Benchmark build impact; keep behind feature flag.
+
+4. **Phase 4 – Documentation & Adoption**
+   - Document env vars/config file usage.
+   - Update KB pipeline guide in README/docs.
+   - Gather feedback, iterate on defaults.
+
+## 12. Testing Strategy
+
+- Unit tests per chunker verifying chunk counts, metadata integrity, and fallback logic.
+- Golden-file tests comparing chunk outputs for representative code/docs.
+- Integration test running pipeline on sample repo (like Dazzle) ensuring no timeouts or memory blowups.
+- Benchmark harness to track runtime vs. file size.
+
+## 13. Open Questions
+
+- Do we need per-language registries (e.g., `.py` vs `.pyi`)?
+- Should summarization integrate directly into pipeline or remain in KB build script?
+- How aggressively should we cache chunk results (hash by content) to avoid recomputation when files don’t change?
+- How do we expose chunk metadata in downstream tools (UI, Slack bot) for debugging?
+
+---
+
+**Next Steps:**
+- Finalize `ChunkerConfig` shape and default values.
+- Implement Phase 1 (sliding window + pipeline scaffolding).
+- Write tests ensuring no regression against current KB build.
+- Incrementally add higher-level chunkers in Phase 2.

chunky_files-0.2.0/docs/index.rst

@@ -0,0 +1,22 @@
+.. chunky documentation master file
+
+Welcome to Chunky's documentation!
+=================================
+
+Chunky provides modular chunking primitives tailored for heterogeneous scientific repositories.
+It is designed to serve both the Nancy Brain knowledge-base pipeline and any external RAG
+pipelines that need deterministic, metadata-rich chunks.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents
+
+   overview
+   api
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

chunky_files-0.2.0/docs/overview.rst

@@ -0,0 +1,50 @@
+Overview
+========
+
+Chunky exposes a modular pipeline for converting heterogeneous project artefacts into
+well-behaved text chunks. The pipeline is language-aware, pluggable, and ready for
+Nancy Brain's MCP-backed retrieval workflows.
+
+.. note::
+   The implementation is in active development. See ``SEMANTIC_CHUNKER.md`` for the full
+   design document and roadmap.
+
+Getting Started
+---------------
+
+Install the package from source:
+
+.. code-block:: bash
+
+   git clone https://github.com/AmberLee2427/chunky.git
+   cd chunky
+   pip install .
+
+For development work and documentation builds:
+
+.. code-block:: bash
+
+   pip install -e ".[dev,docs]"
+
+First chunks via the pipeline:
+
+.. code-block:: python
+
+   from pathlib import Path
+
+   from chunky import ChunkPipeline, ChunkerConfig
+
+   pipeline = ChunkPipeline()
+   config = ChunkerConfig(lines_per_chunk=80, line_overlap=10)
+   chunks = pipeline.chunk_file(Path("/path/to/file.py"), config=config)
+
+   for chunk in chunks:
+       print(chunk.chunk_id, chunk.metadata["line_start"], chunk.metadata["line_end"])
+
+Roadmap
+-------
+
+* Phase 1: infrastructure scaffolding and sliding-window baseline.
+* Phase 2: language-specific chunkers (Python, Markdown, JSON/YAML, notebooks).
+* Phase 3: semantic/embedding-driven chunking.
+* Phase 4: documentation, benchmarks, and Nancy Brain integration.

chunky_files-0.2.0/pyproject.toml

@@ -0,0 +1,111 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "chunky-files"
+version = "0.2.0"
+description = "Semantic chunking utilities for scientific code and documentation corpora."
+readme = "README.md"
+authors = [
+  { name = "Nancy Brain Contributors" }
+]
+license = { file = "LICENSE" }
+requires-python = ">=3.8"
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Text Processing :: Linguistic",
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=7",
+  "pytest-cov>=4",
+  "coverage[toml]>=7",
+  "ruff>=0.6",
+  "bump-my-version>=0.6",
+  "build"
+]
+docs = [
+  "sphinx>=7",
+  "myst-parser>=2",
+  "furo>=2024.0.0",
+]
+
+[project.urls]
+Home = "https://github.com/AmberLee2427/chunky"
+Documentation = "https://chunky.readthedocs.io/"
+Issues = "https://github.com/AmberLee2427/chunky/issues"
+
+[tool.hatch.version]
+path = "src/chunky/__about__.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "src/**",
+  "docs/**",
+  "README.md",
+  "LICENSE",
+  "CHANGELOG.md",
+  "pyproject.toml",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/chunky"]
+
+[tool.hatch.envs.default]
+dependencies = [
+  "pytest",
+]
+
+[tool.pytest.ini_options]
+minversion = "7.0"
+addopts = "-ra --showlocals --strict-markers --strict-config"
+testpaths = [
+  "tests",
+]
+
+
+[tool.ruff]
+line-length = 100
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B"]
+ignore = ["E203"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.bumpversion]
+current_version = "0.2.0"
+commit = true
+message = "chore: bump version to {new_version}"
+
+[[tool.bumpversion.files]]
+filename = "pyproject.toml"
+search = "version = \"{current_version}\""
+replace = "version = \"{new_version}\""
+
+[[tool.bumpversion.files]]
+filename = "src/chunky/__about__.py"
+search = "__version__ = \"{current_version}\""
+replace = "__version__ = \"{new_version}\""
+ 
+[[tool.bumpversion.files]]
+filename = "CHANGELOG.md"
+search = "## [Unreleased]"
+replace = "## [Unreleased]\n\n## [{new_version}] - TBD"

chunky_files-0.2.0/src/chunky/__about__.py

@@ -0,0 +1,5 @@
+"""Project metadata."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.2.0"

chunky_files-0.2.0/src/chunky/__init__.py

@@ -0,0 +1,13 @@
+"""Chunky: semantic chunking utilities for heterogeneous repositories."""
+
+from .__about__ import __version__
+from .pipeline import ChunkPipeline
+from .types import Chunk, ChunkerConfig, Document
+
+__all__ = [
+    "__version__",
+    "ChunkPipeline",
+    "Chunk",
+    "ChunkerConfig",
+    "Document",
+]

chunky_files-0.2.0/src/chunky/chunkers/__init__.py

@@ -0,0 +1,5 @@
+"""Built-in chunker implementations."""
+
+from .fallback import SlidingWindowChunker
+
+__all__ = ["SlidingWindowChunker"]

chunky_files-0.2.0/src/chunky/chunkers/fallback.py

@@ -0,0 +1,112 @@
+"""Sliding window fallback chunker."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ..core import Chunker
+from ..types import Chunk, ChunkerConfig, Document
+
+
+class SlidingWindowChunker(Chunker):
+    """Chunker that produces fixed-size line windows with optional overlap."""
+
+    def chunk(self, document: Document, config: ChunkerConfig) -> list[Chunk]:
+        lines = document.content.splitlines()
+        window = config.clamp_lines(config.lines_per_chunk)
+        overlap = config.clamp_overlap(config.line_overlap, window)
+
+        if not lines:
+            chunk_id = self._build_chunk_id(document.path, 0)
+            return [
+                Chunk(
+                    chunk_id=chunk_id,
+                    text="",
+                    source_document=document.path,
+                    metadata=self._chunk_metadata(
+                        chunk_index=0,
+                        line_start=0,
+                        line_end=0,
+                        span_start=0,
+                        span_end=0,
+                        config=config,
+                    ),
+                )
+            ]
+
+        chunks: list[Chunk] = []
+        line_count = len(lines)
+        # Pre-compute character offsets once to avoid quadratic scans.
+        line_starts: list[int] = []
+        line_ends: list[int] = []
+        cursor = 0
+        for idx, line in enumerate(lines):
+            if idx > 0:
+                cursor += 1  # newline preceding this line
+            line_starts.append(cursor)
+            cursor += len(line)
+            line_ends.append(cursor)
+
+        start_line = 0
+        chunk_index = 0
+
+        while start_line < line_count:
+            previous_start = start_line
+            end_line = min(start_line + window, line_count)
+            text = "\n".join(lines[start_line:end_line])
+            chunk_id = self._build_chunk_id(document.path, chunk_index)
+            metadata = self._chunk_metadata(
+                chunk_index=chunk_index,
+                line_start=start_line + 1,
+                line_end=end_line,
+                span_start=line_starts[start_line],
+                span_end=line_ends[end_line - 1],
+                config=config,
+            )
+
+            chunks.append(
+                Chunk(
+                    chunk_id=chunk_id,
+                    text=text,
+                    source_document=document.path,
+                    metadata=metadata,
+                )
+            )
+
+            chunk_index += 1
+            if config.max_chunks and chunk_index >= config.max_chunks:
+                break
+
+            if end_line >= line_count:
+                break
+
+            next_start = end_line - overlap
+            if next_start <= previous_start:
+                next_start = end_line
+            start_line = next_start
+
+        return chunks
+
+    @staticmethod
+    def _build_chunk_id(path: Path, index: int) -> str:
+        return f"{path}::chunk-{index}"
+
+    @staticmethod
+    def _chunk_metadata(
+        chunk_index: int,
+        line_start: int,
+        line_end: int,
+        span_start: int,
+        span_end: int,
+        config: ChunkerConfig,
+    ) -> dict[str, int | str]:
+        metadata: dict[str, int | str] = {
+            "chunk_index": chunk_index,
+            "line_start": line_start,
+            "line_end": line_end,
+            "span_start": span_start,
+            "span_end": span_end,
+        }
+        if config.metadata:
+            metadata.update(config.metadata)
+        return metadata

chunky_files-0.2.0/src/chunky/core.py

@@ -0,0 +1,22 @@
+"""Core interfaces and exceptions for chunkers."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from .types import Chunk, ChunkerConfig, Document
+
+
+class ChunkingError(RuntimeError):
+    """Raised when a chunker cannot process the provided document."""
+
+
+@runtime_checkable
+class Chunker(Protocol):
+    """Protocol implemented by all chunkers."""
+
+    def chunk(self, document: Document, config: ChunkerConfig) -> list[Chunk]:
+        """Return a list of chunks for the given document."""
+
+
+__all__ = ["ChunkingError", "Chunker"]

chunky_files-0.2.0/src/chunky/loaders.py

@@ -0,0 +1,29 @@
+"""Document loaders."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from .types import Document
+
+
+class DocumentLoader(Protocol):
+    """Protocol for converting files into :class:`Document` instances."""
+
+    def load(self, path: Path) -> Document:
+        ...
+
+
+class FileSystemLoader:
+    """Loader that reads text files from disk."""
+
+    def load(self, path: Path) -> Document:
+        content = path.read_text(encoding="utf-8")
+        return Document(path=path, content=content)
+
+
+DEFAULT_LOADER = FileSystemLoader()
+
+
+__all__ = ["DocumentLoader", "FileSystemLoader", "DEFAULT_LOADER"]

chunky_files-0.2.0/src/chunky/pipeline.py

@@ -0,0 +1,59 @@
+"""High-level orchestration for chunking documents."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from .chunkers import SlidingWindowChunker
+from .loaders import DEFAULT_LOADER, DocumentLoader
+from .registry import DEFAULT_REGISTRY, ChunkerRegistry
+from .types import Chunk, ChunkerConfig, Document
+
+
+class ChunkPipeline:
+    """Pipeline that orchestrates document loading and chunking."""
+
+    def __init__(
+        self,
+        *,
+        registry: Optional[ChunkerRegistry] = None,
+        loader: Optional[DocumentLoader] = None,
+    ) -> None:
+        self.registry = registry or DEFAULT_REGISTRY
+        self.loader = loader or DEFAULT_LOADER
+        self._ensure_fallback()
+
+    def chunk_file(
+        self,
+        path: Path | str,
+        *,
+        config: Optional[ChunkerConfig] = None,
+    ) -> list[Chunk]:
+        """Chunk a file from disk."""
+
+        config = config or ChunkerConfig()
+        document = self.loader.load(Path(path))
+        chunker = self.registry.get(document.path)
+        return chunker.chunk(document, config)
+
+    def chunk_documents(
+        self,
+        documents: list[Document],
+        *,
+        config: Optional[ChunkerConfig] = None,
+    ) -> list[Chunk]:
+        """Chunk pre-loaded documents."""
+
+        config = config or ChunkerConfig()
+        chunks: list[Chunk] = []
+        for document in documents:
+            chunker = self.registry.get(document.path)
+            chunks.extend(chunker.chunk(document, config))
+        return chunks
+
+    def _ensure_fallback(self) -> None:
+        try:
+            self.registry.get(Path("__dummy__"))
+        except KeyError:
+            self.registry.set_fallback(SlidingWindowChunker())

chunky_files-0.2.0/src/chunky/registry.py

@@ -0,0 +1,60 @@
+"""Chunker registry responsible for resolving the appropriate implementation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable, MutableMapping
+
+from .core import Chunker
+
+
+class ChunkerRegistry:
+    """Runtime registry mapping file extensions to chunkers."""
+
+    def __init__(self) -> None:
+        self._registry: MutableMapping[str, Chunker] = {}
+        self._fallback: Chunker | None = None
+
+    def register(
+        self,
+        extensions: Iterable[str] | str,
+        chunker: Chunker,
+        *,
+        overwrite: bool = False,
+    ) -> None:
+        """Register a chunker for one or more extensions."""
+
+        if isinstance(extensions, str):
+            extensions = [extensions]
+
+        for ext in extensions:
+            key = self._normalize(ext)
+            if not overwrite and key in self._registry:
+                raise ValueError(f"Chunker already registered for extension '{ext}'")
+            self._registry[key] = chunker
+
+    def set_fallback(self, chunker: Chunker) -> None:
+        """Set the fallback chunker used for unknown extensions."""
+
+        self._fallback = chunker
+
+    def get(self, path: Path) -> Chunker:
+        """Return the chunker associated with the file path or the fallback."""
+
+        suffix = self._normalize(path.suffix or "")
+        chunker = self._registry.get(suffix)
+        if chunker is not None:
+            return chunker
+        if self._fallback is None:
+            raise KeyError(f"No chunker registered for {suffix!r} and no fallback configured")
+        return self._fallback
+
+    @staticmethod
+    def _normalize(extension: str) -> str:
+        return extension.lower().lstrip(".")
+
+
+DEFAULT_REGISTRY = ChunkerRegistry()
+
+
+__all__ = ["ChunkerRegistry", "DEFAULT_REGISTRY"]

chunky_files-0.2.0/src/chunky/types.py

@@ -0,0 +1,49 @@
+"""Core data structures for the semantic chunking pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class Document:
+    """Normalized representation of a file to be chunked."""
+
+    path: Path
+    content: str
+    language: Optional[str] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Chunk:
+    """A chunk of text ready for downstream indexing."""
+
+    chunk_id: str
+    text: str
+    source_document: Path
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class ChunkerConfig:
+    """Configuration shared across chunkers."""
+
+    max_chars: int = 2000
+    lines_per_chunk: int = 120
+    line_overlap: int = 20
+    max_chunks: Optional[int] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def clamp_lines(self, lines: int) -> int:
+        """Clamp the requested line count to a sensible positive value."""
+
+        return max(1, lines)
+
+    def clamp_overlap(self, overlap: int, window: int) -> int:
+        """Clamp overlap so it cannot exceed the window size."""
+
+        overlap = max(0, overlap)
+        return min(overlap, max(0, window - 1))