krira-augment 2.0.4__tar.gz

krira_augment-2.0.4/Cargo.lock

@@ -0,0 +1,400 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
+name = "csv"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde_core",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+
+[[package]]
+name = "krira_augment"
+version = "2.0.4"
+dependencies = [
+ "csv",
+ "lazy_static",
+ "memmap2",
+ "pyo3",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
+name = "libc"
+version = "0.2.179"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f"
+
+[[package]]
+name = "memchr"
+version = "2.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
+
+[[package]]
+name = "memmap2"
+version = "0.9.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rayon"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "regex"
+version = "1.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "ryu"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984"
+
+[[package]]
+name = "serde"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.148"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3084b546a1dd6289475996f182a22aba973866ea8e8b02c51d9f46b1336a22da"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.112"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21f182278bf2d2bcb3c88b1b08a37df029d71ce3d3ae26168e3c653b213b99d4"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
+
+[[package]]
+name = "zmij"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "317f17ff091ac4515f17cc7a190d2769a8c9a96d227de5d64b500b01cda8f2cd"

krira_augment-2.0.4/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "krira_augment"
+version = "2.0.4"
+edition = "2021"
+authors = ["Krira Labs <contact@kriralabs.com>"]
+description = "Production-grade document chunking library for RAG systems"
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/Krira-Labs/krira-chunker"
+
+[lib]
+name = "krira_augment"
+crate-type = ["cdylib"]
+
+[dependencies]
+# PyO3 for Python bindings
+pyo3 = { version = "0.23.3", features = ["extension-module"] }
+
+# Parallel processing
+rayon = "1.8"
+
+# Fast file reading (Zero-copy)
+memmap2 = "0.9"
+
+# Regex for cleaning
+regex = "1.10"
+lazy_static = "1.4"
+
+# CSV and JSON
+csv = "1.3"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+# Error handling
+thiserror = "1.0"
+
+# Performance optimization
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+panic = "abort"

krira_augment-2.0.4/PKG-INFO

@@ -0,0 +1,209 @@
+Metadata-Version: 2.4
+Name: krira-augment
+Version: 2.0.4
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: openpyxl>=3.0 ; extra == 'xlsx'
+Requires-Dist: pdfplumber>=0.10 ; extra == 'pdf'
+Requires-Dist: python-docx>=0.8 ; extra == 'docx'
+Requires-Dist: polars>=0.20 ; extra == 'csv'
+Requires-Dist: openpyxl>=3.0 ; extra == 'all'
+Requires-Dist: pdfplumber>=0.10 ; extra == 'all'
+Requires-Dist: python-docx>=0.8 ; extra == 'all'
+Requires-Dist: polars>=0.20 ; extra == 'all'
+Requires-Dist: pytest>=7.0 ; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
+Requires-Dist: black>=23.0 ; extra == 'dev'
+Requires-Dist: mypy>=1.0 ; extra == 'dev'
+Requires-Dist: ruff>=0.1 ; extra == 'dev'
+Provides-Extra: xlsx
+Provides-Extra: pdf
+Provides-Extra: docx
+Provides-Extra: csv
+Provides-Extra: all
+Provides-Extra: dev
+Summary: Production-grade document chunking library for RAG systems - Rust-powered Python library
+Keywords: rag,chunking,nlp,document-processing,ai,rust,pyo3
+Author-email: Krira Labs <contact@kriralabs.com>
+License: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/Krira-Labs/krira-chunker
+Project-URL: Repository, https://github.com/Krira-Labs/krira-chunker
+Project-URL: Documentation, https://github.com/Krira-Labs/krira-chunker#readme
+Project-URL: Issues, https://github.com/Krira-Labs/krira-chunker/issues
+
+# Krira Augment ⚡🦀
+
+**The High-Performance Rust Chunking Engine for RAG Pipelines**
+
+[![PyPI version](https://badge.fury.io/py/krira-augment.svg)](https://badge.fury.io/py/krira-augment)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Rust](https://img.shields.io/badge/Built_with-Rust-orange)](https://www.rust-lang.org/)
+
+**Krira Augment** is a production-grade Python library backed by a highly optimized Rust core. It is designed to replace slow, memory-intensive preprocessing steps in large-scale Retrieval Augmented Generation (RAG) systems.
+
+It processes gigabytes of raw unstructured data (CSV, JSONL, TXT) into high-quality, clean chunks in seconds—utilizing **zero-copy memory mapping** and **parallel CPU execution**.
+
+---
+
+## 🚀 Performance Benchmarks
+
+Benchmarks run on a standard 8-core machine (M2 Air equivalent).
+
+| Dataset Size | Legacy (LangChain/Pandas) | Krira V2 (Rust Core) | Speedup |
+| :--- | :--- | :--- | :--- |
+| **100 MB** | ~45 sec | **~0.8 sec** | **56x** 🚀 |
+| **1 GB** | ~8.0 min | **~12.0 sec** | **40x** 🚀 |
+| **10 GB** | *Timeout / OOM* | **~2.1 min** | **Stable** ✅ |
+
+> **Note:** Krira uses O(1) memory. Processing a 100GB file uses the same amount of RAM as a 10MB file.
+
+---
+
+## 📦 Installation
+
+```bash
+pip install krira-augment
+```
+
+*Requirements: Python 3.8+*
+
+---
+
+## 🛠️ Usage
+
+### 1. Quick Start
+For standard use cases, use the default high-throughput pipeline.
+
+```python
+from krira_augment import Pipeline
+
+# Initialize the pipeline
+pipeline = Pipeline()
+
+# Process a 1GB file in seconds
+stats = pipeline.process(
+    input_path="data/raw_knowledge_base.csv",
+    output_path="data/processed_chunks.jsonl"
+)
+
+print(f"✅ Processing complete chunking job.")
+```
+
+### 2. Advanced Configuration (Professional)
+For production RAG, you need fine-grained control over chunking strategies, overlap, and data cleaning.
+
+```python
+from krira_augment import Pipeline, PipelineConfig, SplitStrategy
+
+# Define a robust configuration
+config = PipelineConfig(
+    # Chunking Strategy
+    chunk_size=512,             # Target characters per chunk
+    chunk_overlap=50,           # Context overlap for better retrieval
+    strategy=SplitStrategy.SMART, # Respects sentence/paragraph boundaries
+    
+    # Data Cleaning Rules (Rust-native regex)
+    clean_html=True,            # Remove <div>, <br>, etc.
+    clean_unicode=True,         # Normalize whitespace and emojis
+    min_chunk_len=20,           # Discard garbage/empty chunks
+    
+    # System Performance
+    threads=8,                  # Force usage of 8 CPU cores
+    batch_size=1000             # Write to disk every 1k chunks (Low RAM usage)
+)
+
+# Initialize with config
+pipeline = Pipeline(config=config)
+
+# Execute
+result = pipeline.process(
+    input_path="large_corpus.csv", 
+    output_path="corpus_vectors.jsonl"
+)
+
+print(f"Job ID: {result.job_id}")
+print(f"Throughput: {result.mb_per_second:.2f} MB/s")
+```
+
+---
+
+## 📄 Output Format
+
+The library outputs standard **JSONL** (JSON Lines), ready for direct ingestion into vector databases (Pinecone, Weaviate, Qdrant).
+
+**`processed_chunks.jsonl`**:
+```json
+{"text": "The mitochondria is the powerhouse...", "metadata": {"source": "doc1.csv", "row": 1, "chunk_index": 0}}
+{"text": "It generates most of the chemical energy...", "metadata": {"source": "doc1.csv", "row": 1, "chunk_index": 1}}
+```
+
+---
+
+## 🏗️ Architecture
+
+Krira differs from standard Python loaders by offloading the entire ETL process to a compiled Rust binary.
+
+1.  **Memory Mapping (mmap):** The file is mapped directly from disk to virtual memory. No loading 1GB CSVs into Python RAM.
+2.  **Rayon Parallelism:** The file is sliced into segments and processed across all available CPU cores simultaneously.
+3.  **Serde Serialization:** Chunks are serialized to JSONL directly on the Rust thread, minimizing Python GIL interaction.
+
+---
+
+## 🤝 Integration Example
+
+Seamlessly integrate with generic Python generators to feed embeddings.
+
+```python
+import json
+import openai
+
+def stream_chunks(jsonl_path):
+    """Yields chunks efficiently for embedding API calls."""
+    with open(jsonl_path, 'r') as f:
+        for line in f:
+            yield json.loads(line)
+
+# Use in your downstream application
+for chunk in stream_chunks("processed_chunks.jsonl"):
+    # Mock embedding call
+    # embedding = openai.Embedding.create(input=chunk['text'])
+    pass
+    
+    # Upsert to Vector DB (e.g., Pinecone)
+    # index.upsert(vectors=[(chunk['id'], embedding, chunk['metadata'])])
+```
+
+---
+
+## 🧑‍💻 Development
+
+If you want to modify the Rust core:
+
+1.  **Clone the repo**
+2.  **Install Maturin** (Rust-Python bridge builder)
+    ```bash
+    pip install maturin
+    ```
+3.  **Build and Install locally**
+    ```bash
+    maturin develop --release
+    ```
+
+---
+
+## License
+
+MIT License.
+