contextiq 0.1.0__tar.gz

contextiq-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ContextIQ Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

contextiq-0.1.0/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: contextiq
+Version: 0.1.0
+Summary: Turn messy files into agent-ready context.
+Author: ContextIQ Contributors
+License-Expression: MIT
+Keywords: rag,agents,llm,ingestion,chunking,search,context
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: docs
+Requires-Dist: python-docx>=1.1.0; extra == "docs"
+Requires-Dist: pypdf>=5.0.0; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Dynamic: license-file
+
+# ContextIQ
+
+ContextIQ turns messy files into agent-ready context.
+
+It is a local-first ingestion pipeline for developers building RAG systems, agent memory layers, document search, and eval datasets. Point it at a folder and it produces clean JSONL and Markdown exports with chunked, traceable content.
+
+## Why it exists
+
+Most AI tooling starts after your data is already clean. Real projects get stuck much earlier:
+
+- PDFs are noisy
+- Word docs lose structure
+- repos and notes mix formats
+- chunks are inconsistent
+- source traceability is easy to lose
+
+ContextIQ focuses on the missing middle: consistent ingestion, chunking, and export.
+
+## Features
+
+- Local-first CLI
+- Recursive file ingestion
+- Built-in support for:
+  - `.txt`, `.md`, `.rst`
+  - `.json`, `.jsonl`
+  - `.csv`, `.tsv`
+  - `.html`, `.htm`
+  - optional `.pdf` via `pypdf`
+  - optional `.docx` via `python-docx`
+- Document-aware chunking
+- Source-preserving metadata
+- JSONL and Markdown exports
+- Run manifest with counts, warnings, and timings
+
+## Quickstart
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install -e .[dev]
+contextiq ingest ./examples --out ./build/context
+```
+
+On Windows PowerShell:
+
+```powershell
+python -m venv .venv
+.venv\Scripts\Activate.ps1
+pip install -e .[dev]
+contextiq ingest .\examples --out .\build\context
+```
+
+## CLI
+
+```bash
+contextiq ingest <path> --out <directory>
+```
+
+Useful flags:
+
+- `--include-ext .md,.txt,.json`
+- `--exclude-glob "*.min.js,*.lock"`
+- `--chunk-size 1200`
+- `--chunk-overlap 150`
+- `--formats jsonl,markdown`
+- `--fail-on-warning`
+
+## Output
+
+`contextiq ingest` writes:
+
+- `documents.jsonl`: normalized source documents
+- `chunks.jsonl`: chunked outputs for RAG/agents
+- `chunks.md`: human-readable review file
+- `manifest.json`: summary of the run
+
+Each chunk preserves:
+
+- source path
+- document id
+- chunk id
+- byte and character ranges when available
+- headings / section hints
+
+## Example
+
+```bash
+contextiq ingest ./docs --out ./dist/context --chunk-size 900 --chunk-overlap 120
+```
+
+## Development
+
+```bash
+pip install -e .[dev]
+pytest
+```
+
+## Roadmap
+
+- embeddings plugin interface
+- vector DB exporters
+- OCR pipeline
+- table extraction
+- citation-aware retrieval benchmarks

contextiq-0.1.0/README.md

@@ -0,0 +1,104 @@
+# ContextIQ
+
+ContextIQ turns messy files into agent-ready context.
+
+It is a local-first ingestion pipeline for developers building RAG systems, agent memory layers, document search, and eval datasets. Point it at a folder and it produces clean JSONL and Markdown exports with chunked, traceable content.
+
+## Why it exists
+
+Most AI tooling starts after your data is already clean. Real projects get stuck much earlier:
+
+- PDFs are noisy
+- Word docs lose structure
+- repos and notes mix formats
+- chunks are inconsistent
+- source traceability is easy to lose
+
+ContextIQ focuses on the missing middle: consistent ingestion, chunking, and export.
+
+## Features
+
+- Local-first CLI
+- Recursive file ingestion
+- Built-in support for:
+  - `.txt`, `.md`, `.rst`
+  - `.json`, `.jsonl`
+  - `.csv`, `.tsv`
+  - `.html`, `.htm`
+  - optional `.pdf` via `pypdf`
+  - optional `.docx` via `python-docx`
+- Document-aware chunking
+- Source-preserving metadata
+- JSONL and Markdown exports
+- Run manifest with counts, warnings, and timings
+
+## Quickstart
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install -e .[dev]
+contextiq ingest ./examples --out ./build/context
+```
+
+On Windows PowerShell:
+
+```powershell
+python -m venv .venv
+.venv\Scripts\Activate.ps1
+pip install -e .[dev]
+contextiq ingest .\examples --out .\build\context
+```
+
+## CLI
+
+```bash
+contextiq ingest <path> --out <directory>
+```
+
+Useful flags:
+
+- `--include-ext .md,.txt,.json`
+- `--exclude-glob "*.min.js,*.lock"`
+- `--chunk-size 1200`
+- `--chunk-overlap 150`
+- `--formats jsonl,markdown`
+- `--fail-on-warning`
+
+## Output
+
+`contextiq ingest` writes:
+
+- `documents.jsonl`: normalized source documents
+- `chunks.jsonl`: chunked outputs for RAG/agents
+- `chunks.md`: human-readable review file
+- `manifest.json`: summary of the run
+
+Each chunk preserves:
+
+- source path
+- document id
+- chunk id
+- byte and character ranges when available
+- headings / section hints
+
+## Example
+
+```bash
+contextiq ingest ./docs --out ./dist/context --chunk-size 900 --chunk-overlap 120
+```
+
+## Development
+
+```bash
+pip install -e .[dev]
+pytest
+```
+
+## Roadmap
+
+- embeddings plugin interface
+- vector DB exporters
+- OCR pipeline
+- table extraction
+- citation-aware retrieval benchmarks

contextiq-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "contextiq"
+version = "0.1.0"
+description = "Turn messy files into agent-ready context."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [
+  { name = "ContextIQ Contributors" }
+]
+keywords = ["rag", "agents", "llm", "ingestion", "chunking", "search", "context"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Libraries",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Text Processing",
+]
+dependencies = []
+
+[project.optional-dependencies]
+docs = ["python-docx>=1.1.0", "pypdf>=5.0.0"]
+dev = ["pytest>=8.0.0"]
+
+[project.scripts]
+contextiq = "contextiq.cli:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "--basetemp=./build/.pytest-tmp"
+cache_dir = "build/.pytest-cache"

contextiq-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

contextiq-0.1.0/src/contextiq/__init__.py

@@ -0,0 +1,5 @@
+"""ContextIQ package."""
+
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

contextiq-0.1.0/src/contextiq/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

contextiq-0.1.0/src/contextiq/chunking.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from .models import Chunk, Document
+from .utils import stable_id
+
+
+def chunk_document(document: Document, chunk_size: int, chunk_overlap: int) -> list[Chunk]:
+    if not document.text:
+        return []
+
+    text = document.text
+    chunks: list[Chunk] = []
+    start = 0
+    index = 0
+
+    while start < len(text):
+        end = min(len(text), start + chunk_size)
+        if end < len(text):
+            candidate = text.rfind("\n\n", start, end)
+            if candidate > start + max(200, chunk_size // 3):
+                end = candidate
+            else:
+                sentence_break = text.rfind(". ", start, end)
+                if sentence_break > start + max(120, chunk_size // 4):
+                    end = sentence_break + 1
+
+        chunk_text = text[start:end].strip()
+        if chunk_text:
+            chunks.append(
+                Chunk(
+                    chunk_id=stable_id(document.doc_id, str(index), str(start), str(end)),
+                    doc_id=document.doc_id,
+                    source_path=document.source_path,
+                    text=chunk_text,
+                    start_char=start,
+                    end_char=end,
+                    section_title=_resolve_section_title(document, start, end),
+                    metadata={"source_type": document.source_type},
+                )
+            )
+
+        if end >= len(text):
+            break
+
+        next_start = max(end - chunk_overlap, start + 1)
+        if next_start <= start:
+            next_start = end
+        start = next_start
+        index += 1
+
+    return chunks
+
+
+def _resolve_section_title(document: Document, start: int, end: int) -> str | None:
+    for section in document.sections:
+        if section.start_char <= start < section.end_char:
+            return section.title
+        if start <= section.start_char < end:
+            return section.title
+    return None

contextiq-0.1.0/src/contextiq/cli.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from .exporters import export_result
+from .pipeline import default_config, run_ingest
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="contextiq", description="Turn messy files into agent-ready context.")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    ingest = subparsers.add_parser("ingest", help="Ingest a directory of files.")
+    ingest.add_argument("path", help="Root directory to ingest")
+    ingest.add_argument("--out", required=True, help="Output directory")
+    ingest.add_argument("--include-ext", help="Comma-separated file extensions to include")
+    ingest.add_argument("--exclude-glob", help="Comma-separated glob patterns to exclude")
+    ingest.add_argument("--chunk-size", type=int, default=1200, help="Target chunk size in characters")
+    ingest.add_argument("--chunk-overlap", type=int, default=150, help="Chunk overlap in characters")
+    ingest.add_argument("--formats", default="jsonl,markdown", help="Comma-separated output formats")
+    ingest.add_argument("--fail-on-warning", action="store_true", help="Exit non-zero if warnings occur")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command != "ingest":
+        return 1
+
+    root = Path(args.path).resolve()
+    out_dir = Path(args.out).resolve()
+    config = default_config(root, out_dir)
+
+    if args.include_ext:
+        config.include_extensions = {normalize_extension(ext) for ext in args.include_ext.split(",") if ext.strip()}
+    if args.exclude_glob:
+        config.exclude_globs = [item.strip() for item in args.exclude_glob.split(",") if item.strip()]
+    config.chunk_size = args.chunk_size
+    config.chunk_overlap = args.chunk_overlap
+    config.formats = {item.strip().lower() for item in args.formats.split(",") if item.strip()}
+    config.fail_on_warning = args.fail_on_warning
+
+    result = run_ingest(config)
+    export_result(out_dir, result, root, config.formats, config.to_manifest_config())
+
+    print(f"Ingested {len(result.documents)} documents into {len(result.chunks)} chunks.")
+    if result.warnings:
+        print(f"Warnings: {len(result.warnings)}")
+    print(f"Output: {out_dir}")
+    return 0
+
+
+def normalize_extension(extension: str) -> str:
+    ext = extension.strip().lower()
+    if not ext:
+        return ext
+    return ext if ext.startswith(".") else f".{ext}"
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

contextiq-0.1.0/src/contextiq/discovery.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+
+DEFAULT_EXTENSIONS = {
+    ".txt",
+    ".md",
+    ".rst",
+    ".json",
+    ".jsonl",
+    ".csv",
+    ".tsv",
+    ".html",
+    ".htm",
+    ".pdf",
+    ".docx",
+}
+
+DEFAULT_EXCLUDES = {
+    ".git",
+    ".hg",
+    ".svn",
+    ".venv",
+    "venv",
+    "node_modules",
+    "__pycache__",
+    "dist",
+    "build",
+}
+
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    include_extensions: set[str]
+    exclude_globs: list[str]
+
+
+def iter_files(root: Path, config: DiscoveryConfig) -> Iterable[Path]:
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        if any(part in DEFAULT_EXCLUDES for part in path.parts):
+            continue
+        if config.include_extensions and path.suffix.lower() not in config.include_extensions:
+            continue
+        if any(path.match(pattern) for pattern in config.exclude_globs):
+            continue
+        yield path

contextiq-0.1.0/src/contextiq/exporters.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import datetime as dt
+from pathlib import Path
+
+from .models import Chunk, IngestResult
+from .utils import ensure_dir, write_json, write_jsonl
+
+
+def export_result(
+    out_dir: Path,
+    result: IngestResult,
+    source_root: Path,
+    formats: set[str],
+    config: dict,
+) -> None:
+    ensure_dir(out_dir)
+    write_jsonl(out_dir / "documents.jsonl", (doc.to_dict() for doc in result.documents))
+    write_jsonl(out_dir / "chunks.jsonl", (chunk.to_dict() for chunk in result.chunks))
+
+    if "markdown" in formats:
+        _write_markdown(out_dir / "chunks.md", result.chunks)
+
+    manifest = {
+        "generated_at": dt.datetime.utcnow().isoformat() + "Z",
+        "source_root": str(source_root),
+        "summary": result.summary(),
+        "warnings": result.warnings,
+        "config": config,
+    }
+    write_json(out_dir / "manifest.json", manifest)
+
+
+def _write_markdown(path: Path, chunks: list[Chunk]) -> None:
+    lines = ["# ContextIQ Chunk Export", ""]
+    for chunk in chunks:
+        lines.append(f"## {chunk.chunk_id}")
+        lines.append("")
+        lines.append(f"- Source: `{chunk.source_path}`")
+        lines.append(f"- Document: `{chunk.doc_id}`")
+        lines.append(f"- Range: `{chunk.start_char}:{chunk.end_char}`")
+        if chunk.section_title:
+            lines.append(f"- Section: `{chunk.section_title}`")
+        lines.append("")
+        lines.append(chunk.text)
+        lines.append("")
+    path.write_text("\n".join(lines), encoding="utf-8")

contextiq-0.1.0/src/contextiq/loaders.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import csv
+import json
+from html.parser import HTMLParser
+from pathlib import Path
+
+from .models import Document, Section
+from .utils import normalize_text, stable_id
+
+
+class PlainTextExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.parts: list[str] = []
+
+    def handle_data(self, data: str) -> None:
+        if data.strip():
+            self.parts.append(data.strip())
+
+    def get_text(self) -> str:
+        return "\n".join(self.parts)
+
+
+def extract_sections(text: str) -> list[Section]:
+    sections: list[Section] = []
+    lines = text.splitlines()
+    cursor = 0
+    heading_positions: list[tuple[str, int, int]] = []
+    for line in lines:
+        stripped = line.strip()
+        line_len = len(line) + 1
+        if stripped.startswith("#"):
+            level = len(stripped) - len(stripped.lstrip("#"))
+            title = stripped[level:].strip() or "Section"
+            heading_positions.append((title, level, cursor))
+        cursor += line_len
+
+    for index, (title, level, start) in enumerate(heading_positions):
+        end = heading_positions[index + 1][2] if index + 1 < len(heading_positions) else len(text)
+        sections.append(Section(title=title, level=level, start_char=start, end_char=end))
+    return sections
+
+
+def load_document(path: Path, root: Path) -> Document:
+    suffix = path.suffix.lower()
+    rel_path = path.relative_to(root).as_posix()
+    warnings: list[str] = []
+
+    if suffix in {".txt", ".md", ".rst"}:
+        text = path.read_text(encoding="utf-8", errors="ignore")
+        source_type = suffix[1:]
+    elif suffix in {".json", ".jsonl"}:
+        text = _load_json_like(path, suffix)
+        source_type = suffix[1:]
+    elif suffix in {".csv", ".tsv"}:
+        text = _load_delimited(path, "\t" if suffix == ".tsv" else ",")
+        source_type = suffix[1:]
+    elif suffix in {".html", ".htm"}:
+        text = _load_html(path)
+        source_type = "html"
+    elif suffix == ".pdf":
+        text, warning = _load_pdf(path)
+        source_type = "pdf"
+        if warning:
+            warnings.append(warning)
+    elif suffix == ".docx":
+        text, warning = _load_docx(path)
+        source_type = "docx"
+        if warning:
+            warnings.append(warning)
+    else:
+        raise ValueError(f"Unsupported file type: {path}")
+
+    normalized = normalize_text(text)
+    return Document(
+        doc_id=stable_id(rel_path),
+        source_path=rel_path,
+        source_type=source_type,
+        text=normalized,
+        metadata={"extension": suffix},
+        sections=extract_sections(normalized),
+        warnings=warnings,
+    )
+
+
+def _load_json_like(path: Path, suffix: str) -> str:
+    if suffix == ".json":
+        data = json.loads(path.read_text(encoding="utf-8", errors="ignore"))
+        return json.dumps(data, indent=2, ensure_ascii=False)
+
+    rows = []
+    with path.open("r", encoding="utf-8", errors="ignore") as handle:
+        for line in handle:
+            stripped = line.strip()
+            if stripped:
+                rows.append(json.loads(stripped))
+    return "\n\n".join(json.dumps(row, indent=2, ensure_ascii=False) for row in rows)
+
+
+def _load_delimited(path: Path, delimiter: str) -> str:
+    with path.open("r", encoding="utf-8", errors="ignore", newline="") as handle:
+        reader = csv.reader(handle, delimiter=delimiter)
+        lines = [" | ".join(cell.strip() for cell in row) for row in reader]
+    return "\n".join(lines)
+
+
+def _load_html(path: Path) -> str:
+    parser = PlainTextExtractor()
+    parser.feed(path.read_text(encoding="utf-8", errors="ignore"))
+    return parser.get_text()
+
+
+def _load_pdf(path: Path) -> tuple[str, str | None]:
+    try:
+        from pypdf import PdfReader
+    except ImportError:
+        return (
+            "",
+            "Skipped PDF content because 'pypdf' is not installed. Install with: pip install contextiq[docs]",
+        )
+
+    reader = PdfReader(str(path))
+    parts = [page.extract_text() or "" for page in reader.pages]
+    return "\n\n".join(parts), None
+
+
+def _load_docx(path: Path) -> tuple[str, str | None]:
+    try:
+        from docx import Document as WordDocument
+    except ImportError:
+        return (
+            "",
+            "Skipped DOCX content because 'python-docx' is not installed. Install with: pip install contextiq[docs]",
+        )
+
+    doc = WordDocument(str(path))
+    return "\n".join(paragraph.text for paragraph in doc.paragraphs), None

contextiq-0.1.0/src/contextiq/models.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any
+
+
+@dataclass(slots=True)
+class Section:
+    title: str
+    level: int
+    start_char: int
+    end_char: int
+
+
+@dataclass(slots=True)
+class Document:
+    doc_id: str
+    source_path: str
+    source_type: str
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    sections: list[Section] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["char_count"] = len(self.text)
+        return payload
+
+
+@dataclass(slots=True)
+class Chunk:
+    chunk_id: str
+    doc_id: str
+    source_path: str
+    text: str
+    start_char: int
+    end_char: int
+    section_title: str | None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = asdict(self)
+        payload["char_count"] = len(self.text)
+        return payload
+
+
+@dataclass(slots=True)
+class IngestResult:
+    documents: list[Document]
+    chunks: list[Chunk]
+    warnings: list[str]
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "document_count": len(self.documents),
+            "chunk_count": len(self.chunks),
+            "warning_count": len(self.warnings),
+        }

contextiq-0.1.0/src/contextiq/pipeline.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .chunking import chunk_document
+from .discovery import DEFAULT_EXTENSIONS, DiscoveryConfig, iter_files
+from .loaders import load_document
+from .models import IngestResult
+
+
+@dataclass(slots=True)
+class IngestConfig:
+    root: Path
+    output_dir: Path
+    include_extensions: set[str]
+    exclude_globs: list[str]
+    chunk_size: int
+    chunk_overlap: int
+    formats: set[str]
+    fail_on_warning: bool = False
+
+    def to_manifest_config(self) -> dict:
+        return {
+            "include_extensions": sorted(self.include_extensions),
+            "exclude_globs": self.exclude_globs,
+            "chunk_size": self.chunk_size,
+            "chunk_overlap": self.chunk_overlap,
+            "formats": sorted(self.formats),
+            "fail_on_warning": self.fail_on_warning,
+        }
+
+
+def default_config(root: Path, output_dir: Path) -> IngestConfig:
+    return IngestConfig(
+        root=root,
+        output_dir=output_dir,
+        include_extensions=set(DEFAULT_EXTENSIONS),
+        exclude_globs=[],
+        chunk_size=1200,
+        chunk_overlap=150,
+        formats={"jsonl", "markdown"},
+        fail_on_warning=False,
+    )
+
+
+def run_ingest(config: IngestConfig) -> IngestResult:
+    discovery = DiscoveryConfig(
+        include_extensions={ext.lower() for ext in config.include_extensions},
+        exclude_globs=config.exclude_globs,
+    )
+    documents = []
+    chunks = []
+    warnings: list[str] = []
+
+    for path in iter_files(config.root, discovery):
+        document = load_document(path, config.root)
+        if not document.text:
+            warnings.extend(f"{document.source_path}: {warning}" for warning in document.warnings)
+            documents.append(document)
+            continue
+
+        documents.append(document)
+        warnings.extend(f"{document.source_path}: {warning}" for warning in document.warnings)
+        chunks.extend(chunk_document(document, config.chunk_size, config.chunk_overlap))
+
+    if config.fail_on_warning and warnings:
+        raise ValueError("Warnings encountered during ingest:\n" + "\n".join(warnings))
+
+    return IngestResult(documents=documents, chunks=chunks, warnings=warnings)

contextiq-0.1.0/src/contextiq/utils.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Iterable
+
+
+WHITESPACE_RE = re.compile(r"[ \t]+")
+BLANKLINE_RE = re.compile(r"\n{3,}")
+
+
+def stable_id(*parts: str) -> str:
+    joined = "::".join(parts)
+    return hashlib.sha1(joined.encode("utf-8")).hexdigest()[:16]
+
+
+def normalize_text(text: str) -> str:
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    text = "\n".join(WHITESPACE_RE.sub(" ", line).rstrip() for line in text.splitlines())
+    return BLANKLINE_RE.sub("\n\n", text).strip()
+
+
+def ensure_dir(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def write_json(path: Path, data: object) -> None:
+    path.write_text(json.dumps(data, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
+
+
+def write_jsonl(path: Path, rows: Iterable[dict]) -> None:
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=True) + "\n")

contextiq-0.1.0/src/contextiq.egg-info/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: contextiq
+Version: 0.1.0
+Summary: Turn messy files into agent-ready context.
+Author: ContextIQ Contributors
+License-Expression: MIT
+Keywords: rag,agents,llm,ingestion,chunking,search,context
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: docs
+Requires-Dist: python-docx>=1.1.0; extra == "docs"
+Requires-Dist: pypdf>=5.0.0; extra == "docs"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Dynamic: license-file
+
+# ContextIQ
+
+ContextIQ turns messy files into agent-ready context.
+
+It is a local-first ingestion pipeline for developers building RAG systems, agent memory layers, document search, and eval datasets. Point it at a folder and it produces clean JSONL and Markdown exports with chunked, traceable content.
+
+## Why it exists
+
+Most AI tooling starts after your data is already clean. Real projects get stuck much earlier:
+
+- PDFs are noisy
+- Word docs lose structure
+- repos and notes mix formats
+- chunks are inconsistent
+- source traceability is easy to lose
+
+ContextIQ focuses on the missing middle: consistent ingestion, chunking, and export.
+
+## Features
+
+- Local-first CLI
+- Recursive file ingestion
+- Built-in support for:
+  - `.txt`, `.md`, `.rst`
+  - `.json`, `.jsonl`
+  - `.csv`, `.tsv`
+  - `.html`, `.htm`
+  - optional `.pdf` via `pypdf`
+  - optional `.docx` via `python-docx`
+- Document-aware chunking
+- Source-preserving metadata
+- JSONL and Markdown exports
+- Run manifest with counts, warnings, and timings
+
+## Quickstart
+
+```bash
+python -m venv .venv
+. .venv/bin/activate
+pip install -e .[dev]
+contextiq ingest ./examples --out ./build/context
+```
+
+On Windows PowerShell:
+
+```powershell
+python -m venv .venv
+.venv\Scripts\Activate.ps1
+pip install -e .[dev]
+contextiq ingest .\examples --out .\build\context
+```
+
+## CLI
+
+```bash
+contextiq ingest <path> --out <directory>
+```
+
+Useful flags:
+
+- `--include-ext .md,.txt,.json`
+- `--exclude-glob "*.min.js,*.lock"`
+- `--chunk-size 1200`
+- `--chunk-overlap 150`
+- `--formats jsonl,markdown`
+- `--fail-on-warning`
+
+## Output
+
+`contextiq ingest` writes:
+
+- `documents.jsonl`: normalized source documents
+- `chunks.jsonl`: chunked outputs for RAG/agents
+- `chunks.md`: human-readable review file
+- `manifest.json`: summary of the run
+
+Each chunk preserves:
+
+- source path
+- document id
+- chunk id
+- byte and character ranges when available
+- headings / section hints
+
+## Example
+
+```bash
+contextiq ingest ./docs --out ./dist/context --chunk-size 900 --chunk-overlap 120
+```
+
+## Development
+
+```bash
+pip install -e .[dev]
+pytest
+```
+
+## Roadmap
+
+- embeddings plugin interface
+- vector DB exporters
+- OCR pipeline
+- table extraction
+- citation-aware retrieval benchmarks

contextiq-0.1.0/src/contextiq.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+LICENSE
+README.md
+pyproject.toml
+src/contextiq/__init__.py
+src/contextiq/__main__.py
+src/contextiq/chunking.py
+src/contextiq/cli.py
+src/contextiq/discovery.py
+src/contextiq/exporters.py
+src/contextiq/loaders.py
+src/contextiq/models.py
+src/contextiq/pipeline.py
+src/contextiq/utils.py
+src/contextiq.egg-info/PKG-INFO
+src/contextiq.egg-info/SOURCES.txt
+src/contextiq.egg-info/dependency_links.txt
+src/contextiq.egg-info/entry_points.txt
+src/contextiq.egg-info/requires.txt
+src/contextiq.egg-info/top_level.txt
+tests/test_chunking.py
+tests/test_pipeline.py

contextiq-0.1.0/src/contextiq.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

contextiq-0.1.0/src/contextiq.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+contextiq = contextiq.cli:main

contextiq-0.1.0/src/contextiq.egg-info/requires.txt

@@ -0,0 +1,7 @@
+
+[dev]
+pytest>=8.0.0
+
+[docs]
+python-docx>=1.1.0
+pypdf>=5.0.0

contextiq-0.1.0/src/contextiq.egg-info/top_level.txt

@@ -0,0 +1 @@
+contextiq

contextiq-0.1.0/tests/test_chunking.py

@@ -0,0 +1,19 @@
+from contextiq.chunking import chunk_document
+from contextiq.models import Document
+
+
+def test_chunk_document_preserves_source_metadata():
+    text = "# Intro\n\n" + ("A" * 700) + "\n\n## Details\n\n" + ("B" * 700)
+    document = Document(
+        doc_id="doc1",
+        source_path="notes/sample.md",
+        source_type="md",
+        text=text,
+    )
+
+    chunks = chunk_document(document, chunk_size=800, chunk_overlap=100)
+
+    assert len(chunks) >= 2
+    assert all(chunk.doc_id == "doc1" for chunk in chunks)
+    assert all(chunk.source_path == "notes/sample.md" for chunk in chunks)
+    assert chunks[0].start_char == 0

contextiq-0.1.0/tests/test_pipeline.py

@@ -0,0 +1,20 @@
+import json
+
+from contextiq.pipeline import default_config, run_ingest
+
+
+def test_pipeline_ingests_directory(tmp_path):
+    docs = tmp_path / "docs"
+    docs.mkdir()
+    (docs / "a.md").write_text("# Title\n\nHello world\n\nSecond paragraph", encoding="utf-8")
+    (docs / "b.txt").write_text("Plain text document", encoding="utf-8")
+    (docs / "data.json").write_text(json.dumps({"name": "contextiq", "kind": "demo"}), encoding="utf-8")
+
+    out_dir = tmp_path / "out"
+    config = default_config(docs, out_dir)
+
+    result = run_ingest(config)
+
+    assert len(result.documents) == 3
+    assert len(result.chunks) >= 3
+    assert result.warnings == []