chromerag 0.1.0__tar.gz

chromerag-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bhargava Chary Peddapudi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chromerag-0.1.0/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Bhargava Chary Peddapudi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

chromerag-0.1.0/PKG-INFO

@@ -0,0 +1,241 @@
+Metadata-Version: 2.4
+Name: chromerag
+Version: 0.1.0
+Summary: ChromeRAG: ingest-time elimination of site template noise for enterprise web RAG.
+Author-email: Bhargava Chary Peddapudi <pedapudibhargav@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/pedapudibhargav/ChromeRAG
+Project-URL: Repository, https://github.com/pedapudibhargav/ChromeRAG.git
+Project-URL: Issues, https://github.com/pedapudibhargav/ChromeRAG/issues
+Project-URL: Documentation, https://pedapudibhargav.github.io/ChromeRAG/
+Keywords: rag,html,markdown,boilerplate,chrome,vector,ingest
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: LICENSE.txt
+Requires-Dist: beautifulsoup4<5,>=4.12.0
+Requires-Dist: lxml<7,>=5.0.0
+Requires-Dist: pydantic<3,>=2.7.0
+Requires-Dist: pyyaml<7,>=6.0.0
+Requires-Dist: tiktoken>=0.7.0
+Requires-Dist: numpy<3,>=1.26.0
+Provides-Extra: dvdf
+Requires-Dist: onnxruntime<2,>=1.17.0; extra == "dvdf"
+Provides-Extra: fetch
+Requires-Dist: httpx<0.29,>=0.27.0; extra == "fetch"
+Provides-Extra: baselines
+Requires-Dist: markdownify>=0.13.0; extra == "baselines"
+Requires-Dist: readability-lxml>=0.8.1; extra == "baselines"
+Requires-Dist: trafilatura>=1.12.0; extra == "baselines"
+Requires-Dist: markitdown>=0.1.0; extra == "baselines"
+Requires-Dist: html2text>=2024.2.26; extra == "baselines"
+Provides-Extra: ui
+Requires-Dist: fastapi>=0.110.0; extra == "ui"
+Requires-Dist: uvicorn>=0.27.0; extra == "ui"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: ruff>=0.6.0; extra == "dev"
+Requires-Dist: httpx<0.29,>=0.27.0; extra == "dev"
+Dynamic: license-file
+
+# ChromeRAG
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
+[![GitHub Pages](https://img.shields.io/badge/docs-GitHub%20Pages-222.svg)](https://pedapudibhargav.github.io/ChromeRAG/)
+
+**HTML → RAG-ready Markdown** that strips site-template chrome (nav, footer, CTAs, cookie banners) while keeping documentation, pricing tables, and article body text.
+
+> Built for **enterprise RAG ingest**, **LLM chunking**, **vector indexing**, and **boilerplate / noise removal** from scraped HTML — not for pixel-perfect web archiving.
+
+| Owns | Does **not** own |
+|---|---|
+| HTML string / file → clean Markdown | Crawling, Playwright, rate limits |
+| Optional site-chrome **learn → extract** (STCE) | Full browser rendering of empty JS shells (warns; caller must render first) |
+| Schema.org → YAML front-matter | Vector DB / embeddings |
+| Precision / coverage priority knobs | Hosted SaaS API |
+
+**Paper:** *ChromeRAG: Ingest-Time Elimination of Site Template Noise for Enterprise Web RAG*
+**Release:** [`v0.1.0`](https://github.com/pedapudibhargav/ChromeRAG/tree/v0.1.0)  
+**Author:** [Bhargava Chary Peddapudi](https://orcid.org/0009-0002-8523-8415)
+
+---
+
+## Why ChromeRAG (vs MarkItDown / Trafilatura)?
+
+| Tool | Best at | Gap for corporate web RAG |
+|------|---------|---------------------------|
+| **MarkItDown** | Office/PDF/HTML → Markdown for LLMs | Keeps a lot of page chrome; not tuned to strip SaaS nav/footer |
+| **Trafilatura** | News/article main-content | Weaker on docs hubs, pricing matrices, marketing shells |
+| **Readability** | Article extraction | Often drops tables / side content needed for RAG |
+| **ChromeRAG** | Ingest-time chrome elimination + schema + tables | Focused HTML→RAG Markdown (fetch stays in your crawler) |
+
+---
+
+## Benchmarks (public corpus)
+
+**Deterministic metrics** (primary claims) — independent of any single extractor:
+
+| Metric | Meaning | Better |
+|--------|---------|--------|
+| **Recall** (`content_recall`) | Fraction of main/article text anchors kept | Higher |
+| **Noise ret** (`noise_retention`) | Fraction of nav/footer chrome anchors kept | Lower |
+| **Fbal** (`f_balanced`) | Balance of high recall + low noise | Higher |
+
+**Baselines compared:** ChromeRAG (balanced / coverage / precision), Trafilatura, Readability, **MarkItDown**, markdownify, html2text, BeautifulSoup text.
+
+**Corpus:** **373** URLs listed in `poc/corpus_urls.json` across `docs`, `pricing`, `marketing`, `wiki`, `hub`, `news`, `article`, `cloud`. Latest run fetched **277** HTML pages; **238** were scoreable (DOM content-anchor coverage ≥ 0.05; same fixed cohort for every tool); **39** thin pages are excluded from leaderboard *means* so empty JS shells are not silently averaged into SOTA claims.
+
+**Latest leaderboard (238 scoreable pages):**
+
+| Method | Recall ↑ | Noise ↓ | Fbal ↑ |
+|--------|--------:|--------:|-------:|
+| chromerag_coverage | 0.701 | 0.005 | **0.800** |
+| chromerag (balanced) | 0.678 | 0.005 | 0.783 |
+| trafilatura | 0.662 | 0.019 | 0.752 |
+| markitdown | 0.702 | 0.261 | 0.695 |
+| readability | 0.456 | 0.016 | 0.535 |
+
+Full tables + per-category breakdown: [Results](https://pedapudibhargav.github.io/ChromeRAG/results.html) · `docs/data/corpus_comparison_summary.md`.
+
+**Optional LLM judge** (secondary): stratified sample scored 1–5 on content keep / noise strip / structure when `OPENAI_API_KEY` is set (`python -m poc.run_llm_eval`). Does not replace Recall/Noise/Fbal.
+
+---
+
+## Quick start
+
+### From PyPI (recommended)
+
+```bash
+pip install chromerag
+# Optional: ONNX MiniLM density pruning
+# pip install "chromerag[dvdf]"
+```
+
+### From source
+
+```bash
+git clone https://github.com/pedapudibhargav/ChromeRAG.git
+cd ChromeRAG
+python3 -m venv .venv
+source .venv/bin/activate   # Windows: .venv\Scripts\activate
+
+# Core library + CLI
+pip install -e .
+
+# Optional: comparison baselines + tests
+pip install -e ".[dev,baselines]"
+```
+
+### CLI
+
+```bash
+# Single page
+chromerag extract path/to/page.html -o out.md --priority balanced --json-meta
+
+# Priorities: precision | balanced | coverage
+chromerag extract page.html -o clean.md --priority coverage
+
+# Thin / JS-shell HTML prints WARNING on stderr (caller must Playwright-render first)
+# chromerag extract spa.html -o out.md --fail-on-thin   # exit 3 if thin
+
+# Learn site chrome across a folder, then batch-extract
+chromerag learn data/raw -o data/chrome_models/site.json --min-pages 3
+chromerag batch data/raw -o data/outputs --chrome-model data/chrome_models/site.json
+```
+
+### Python
+
+```python
+from chromerag import ChromeRAG, PipelineConfig, ContentPriority
+
+html = open("page.html", encoding="utf-8").read()
+result = ChromeRAG(
+    config=PipelineConfig.from_priority(ContentPriority.BALANCED, enable_dvdf=False)
+).extract(html, url="https://example.com/docs")
+
+print(result.markdown)       # RAG-ready Markdown (+ YAML front-matter when Schema.org present)
+print(result.front_matter)   # dict
+print(result.tokens_estimate)
+print(result.warnings)       # e.g. JS shell → render with Playwright first
+```
+
+---
+
+## Reproduce the evaluation
+
+```bash
+# 1) Fetch / refresh the public corpus (uses poc/corpus_urls.json)
+python -m poc.run_corpus_comparison
+
+# 2) Export tables + JSON into docs/ for GitHub Pages
+python -m poc.export_site_results
+
+# 3) Revalidate published numbers + classify thin pages
+python scripts/revalidate_corpus.py
+
+# 4) Optional LLM subset judge (needs OPENAI_API_KEY)
+python -m poc.run_llm_eval --sample 40
+
+# 5) Unit tests + regenerate docs/tests.html
+./scripts/build_docs.sh
+```
+
+Skip re-fetch if HTML is already under `data/raw/`:
+
+```bash
+python -m poc.run_corpus_comparison --no-fetch
+```
+
+---
+
+## Docs site (GitHub Pages)
+
+Static files live in [`docs/`](docs/) (relative links only).
+
+CI (`.github/workflows/pages.yml`) on every push to **`main`**:
+
+1. runs `pytest`
+2. rebuilds `docs/tests.html` + exports corpus results
+3. deploys `docs/` → **`gh-pages`** branch
+
+Enable once: **Settings → Pages → Deploy from a branch → `gh-pages` / (root)**.  
+Site: https://pedapudibhargav.github.io/ChromeRAG/
+
+---
+
+## Project layout
+
+```
+src/chromerag/     # library (HTML in → Markdown out)
+poc/               # fetch, baselines, corpus comparison, LLM eval (not required at runtime)
+docs/              # GitHub Pages site + published metrics
+tests/             # unit tests
+papers/softwarex/  # SoftwareX manuscript draft
+```
+
+---
+
+
+## Publishing (SoftwareX)
+
+- Manuscript draft (Markdown): [`papers/softwarex/SOFTWAREX_DRAFT.md`](papers/softwarex/SOFTWAREX_DRAFT.md)
+- Filled official OSP Word manuscript: [`papers/softwarex/ChromeRAG_SoftwareX_OSP.docx`](papers/softwarex/ChromeRAG_SoftwareX_OSP.docx)
+- Figures: [`papers/softwarex/figures/`](papers/softwarex/figures/)
+- Limitations (JS shells / scoreable filter): [docs/limitations.html](docs/limitations.html)
+- Draft self-check: `python scripts/check_softwarex_draft.py`
+- Highlights / APC / refs explained: [`papers/softwarex/WHAT_THESE_MEAN.md`](papers/softwarex/WHAT_THESE_MEAN.md)
+- Optional screencast shot list: [`papers/softwarex/SCREENCAST.md`](papers/softwarex/SCREENCAST.md)
+
+SoftwareX APC (journal OA fee) is paid **only after acceptance**, not at submission.
+
+
+## License
+
+MIT — see [LICENSE](LICENSE) / [LICENSE.txt](LICENSE.txt) (SoftwareX naming) · also `Licence.txt`.

chromerag-0.1.0/README.md

@@ -0,0 +1,195 @@
+# ChromeRAG
+
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
+[![GitHub Pages](https://img.shields.io/badge/docs-GitHub%20Pages-222.svg)](https://pedapudibhargav.github.io/ChromeRAG/)
+
+**HTML → RAG-ready Markdown** that strips site-template chrome (nav, footer, CTAs, cookie banners) while keeping documentation, pricing tables, and article body text.
+
+> Built for **enterprise RAG ingest**, **LLM chunking**, **vector indexing**, and **boilerplate / noise removal** from scraped HTML — not for pixel-perfect web archiving.
+
+| Owns | Does **not** own |
+|---|---|
+| HTML string / file → clean Markdown | Crawling, Playwright, rate limits |
+| Optional site-chrome **learn → extract** (STCE) | Full browser rendering of empty JS shells (warns; caller must render first) |
+| Schema.org → YAML front-matter | Vector DB / embeddings |
+| Precision / coverage priority knobs | Hosted SaaS API |
+
+**Paper:** *ChromeRAG: Ingest-Time Elimination of Site Template Noise for Enterprise Web RAG*
+**Release:** [`v0.1.0`](https://github.com/pedapudibhargav/ChromeRAG/tree/v0.1.0)  
+**Author:** [Bhargava Chary Peddapudi](https://orcid.org/0009-0002-8523-8415)
+
+---
+
+## Why ChromeRAG (vs MarkItDown / Trafilatura)?
+
+| Tool | Best at | Gap for corporate web RAG |
+|------|---------|---------------------------|
+| **MarkItDown** | Office/PDF/HTML → Markdown for LLMs | Keeps a lot of page chrome; not tuned to strip SaaS nav/footer |
+| **Trafilatura** | News/article main-content | Weaker on docs hubs, pricing matrices, marketing shells |
+| **Readability** | Article extraction | Often drops tables / side content needed for RAG |
+| **ChromeRAG** | Ingest-time chrome elimination + schema + tables | Focused HTML→RAG Markdown (fetch stays in your crawler) |
+
+---
+
+## Benchmarks (public corpus)
+
+**Deterministic metrics** (primary claims) — independent of any single extractor:
+
+| Metric | Meaning | Better |
+|--------|---------|--------|
+| **Recall** (`content_recall`) | Fraction of main/article text anchors kept | Higher |
+| **Noise ret** (`noise_retention`) | Fraction of nav/footer chrome anchors kept | Lower |
+| **Fbal** (`f_balanced`) | Balance of high recall + low noise | Higher |
+
+**Baselines compared:** ChromeRAG (balanced / coverage / precision), Trafilatura, Readability, **MarkItDown**, markdownify, html2text, BeautifulSoup text.
+
+**Corpus:** **373** URLs listed in `poc/corpus_urls.json` across `docs`, `pricing`, `marketing`, `wiki`, `hub`, `news`, `article`, `cloud`. Latest run fetched **277** HTML pages; **238** were scoreable (DOM content-anchor coverage ≥ 0.05; same fixed cohort for every tool); **39** thin pages are excluded from leaderboard *means* so empty JS shells are not silently averaged into SOTA claims.
+
+**Latest leaderboard (238 scoreable pages):**
+
+| Method | Recall ↑ | Noise ↓ | Fbal ↑ |
+|--------|--------:|--------:|-------:|
+| chromerag_coverage | 0.701 | 0.005 | **0.800** |
+| chromerag (balanced) | 0.678 | 0.005 | 0.783 |
+| trafilatura | 0.662 | 0.019 | 0.752 |
+| markitdown | 0.702 | 0.261 | 0.695 |
+| readability | 0.456 | 0.016 | 0.535 |
+
+Full tables + per-category breakdown: [Results](https://pedapudibhargav.github.io/ChromeRAG/results.html) · `docs/data/corpus_comparison_summary.md`.
+
+**Optional LLM judge** (secondary): stratified sample scored 1–5 on content keep / noise strip / structure when `OPENAI_API_KEY` is set (`python -m poc.run_llm_eval`). Does not replace Recall/Noise/Fbal.
+
+---
+
+## Quick start
+
+### From PyPI (recommended)
+
+```bash
+pip install chromerag
+# Optional: ONNX MiniLM density pruning
+# pip install "chromerag[dvdf]"
+```
+
+### From source
+
+```bash
+git clone https://github.com/pedapudibhargav/ChromeRAG.git
+cd ChromeRAG
+python3 -m venv .venv
+source .venv/bin/activate   # Windows: .venv\Scripts\activate
+
+# Core library + CLI
+pip install -e .
+
+# Optional: comparison baselines + tests
+pip install -e ".[dev,baselines]"
+```
+
+### CLI
+
+```bash
+# Single page
+chromerag extract path/to/page.html -o out.md --priority balanced --json-meta
+
+# Priorities: precision | balanced | coverage
+chromerag extract page.html -o clean.md --priority coverage
+
+# Thin / JS-shell HTML prints WARNING on stderr (caller must Playwright-render first)
+# chromerag extract spa.html -o out.md --fail-on-thin   # exit 3 if thin
+
+# Learn site chrome across a folder, then batch-extract
+chromerag learn data/raw -o data/chrome_models/site.json --min-pages 3
+chromerag batch data/raw -o data/outputs --chrome-model data/chrome_models/site.json
+```
+
+### Python
+
+```python
+from chromerag import ChromeRAG, PipelineConfig, ContentPriority
+
+html = open("page.html", encoding="utf-8").read()
+result = ChromeRAG(
+    config=PipelineConfig.from_priority(ContentPriority.BALANCED, enable_dvdf=False)
+).extract(html, url="https://example.com/docs")
+
+print(result.markdown)       # RAG-ready Markdown (+ YAML front-matter when Schema.org present)
+print(result.front_matter)   # dict
+print(result.tokens_estimate)
+print(result.warnings)       # e.g. JS shell → render with Playwright first
+```
+
+---
+
+## Reproduce the evaluation
+
+```bash
+# 1) Fetch / refresh the public corpus (uses poc/corpus_urls.json)
+python -m poc.run_corpus_comparison
+
+# 2) Export tables + JSON into docs/ for GitHub Pages
+python -m poc.export_site_results
+
+# 3) Revalidate published numbers + classify thin pages
+python scripts/revalidate_corpus.py
+
+# 4) Optional LLM subset judge (needs OPENAI_API_KEY)
+python -m poc.run_llm_eval --sample 40
+
+# 5) Unit tests + regenerate docs/tests.html
+./scripts/build_docs.sh
+```
+
+Skip re-fetch if HTML is already under `data/raw/`:
+
+```bash
+python -m poc.run_corpus_comparison --no-fetch
+```
+
+---
+
+## Docs site (GitHub Pages)
+
+Static files live in [`docs/`](docs/) (relative links only).
+
+CI (`.github/workflows/pages.yml`) on every push to **`main`**:
+
+1. runs `pytest`
+2. rebuilds `docs/tests.html` + exports corpus results
+3. deploys `docs/` → **`gh-pages`** branch
+
+Enable once: **Settings → Pages → Deploy from a branch → `gh-pages` / (root)**.  
+Site: https://pedapudibhargav.github.io/ChromeRAG/
+
+---
+
+## Project layout
+
+```
+src/chromerag/     # library (HTML in → Markdown out)
+poc/               # fetch, baselines, corpus comparison, LLM eval (not required at runtime)
+docs/              # GitHub Pages site + published metrics
+tests/             # unit tests
+papers/softwarex/  # SoftwareX manuscript draft
+```
+
+---
+
+
+## Publishing (SoftwareX)
+
+- Manuscript draft (Markdown): [`papers/softwarex/SOFTWAREX_DRAFT.md`](papers/softwarex/SOFTWAREX_DRAFT.md)
+- Filled official OSP Word manuscript: [`papers/softwarex/ChromeRAG_SoftwareX_OSP.docx`](papers/softwarex/ChromeRAG_SoftwareX_OSP.docx)
+- Figures: [`papers/softwarex/figures/`](papers/softwarex/figures/)
+- Limitations (JS shells / scoreable filter): [docs/limitations.html](docs/limitations.html)
+- Draft self-check: `python scripts/check_softwarex_draft.py`
+- Highlights / APC / refs explained: [`papers/softwarex/WHAT_THESE_MEAN.md`](papers/softwarex/WHAT_THESE_MEAN.md)
+- Optional screencast shot list: [`papers/softwarex/SCREENCAST.md`](papers/softwarex/SCREENCAST.md)
+
+SoftwareX APC (journal OA fee) is paid **only after acceptance**, not at submission.
+
+
+## License
+
+MIT — see [LICENSE](LICENSE) / [LICENSE.txt](LICENSE.txt) (SoftwareX naming) · also `Licence.txt`.

chromerag-0.1.0/pyproject.toml

@@ -0,0 +1,77 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "chromerag"
+version = "0.1.0"
+description = "ChromeRAG: ingest-time elimination of site template noise for enterprise web RAG."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "Bhargava Chary Peddapudi", email = "pedapudibhargav@gmail.com" }]
+keywords = [
+  "rag",
+  "html",
+  "markdown",
+  "boilerplate",
+  "chrome",
+  "vector",
+  "ingest",
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Topic :: Text Processing :: Markup :: HTML",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+  "beautifulsoup4>=4.12.0,<5",
+  "lxml>=5.0.0,<7",
+  "pydantic>=2.7.0,<3",
+  "pyyaml>=6.0.0,<7",
+  "tiktoken>=0.7.0",
+  "numpy>=1.26.0,<3",
+]
+
+[project.optional-dependencies]
+dvdf = [
+  "onnxruntime>=1.17.0,<2",
+]
+# Fetch stays optional — ChromeRAG core is HTML-in / Markdown-out.
+fetch = [
+  "httpx>=0.27.0,<0.29",
+]
+baselines = [
+  "markdownify>=0.13.0",
+  "readability-lxml>=0.8.1",
+  "trafilatura>=1.12.0",
+  "markitdown>=0.1.0",
+  "html2text>=2024.2.26",
+]
+ui = [
+  "fastapi>=0.110.0",
+  "uvicorn>=0.27.0",
+]
+dev = [
+  "pytest>=8.0.0",
+  "ruff>=0.6.0",
+  "httpx>=0.27.0,<0.29",
+]
+
+[project.scripts]
+chromerag = "chromerag.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/pedapudibhargav/ChromeRAG"
+Repository = "https://github.com/pedapudibhargav/ChromeRAG.git"
+Issues = "https://github.com/pedapudibhargav/ChromeRAG/issues"
+Documentation = "https://pedapudibhargav.github.io/ChromeRAG/"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+chromerag = ["assets/*.json", "assets/*.txt"]

chromerag-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

chromerag-0.1.0/src/chromerag/__init__.py

@@ -0,0 +1,37 @@
+"""ChromeRAG — Ingest-time elimination of site template noise for enterprise web RAG.
+
+Core contract: HTML string in → RAG-ready Markdown out.
+Fetching/crawling is intentionally out of scope (use your crawler / poc harness).
+"""
+
+from chromerag.batch import learn_then_extract
+from chromerag.config import ContentPriority, PageType, PipelineConfig, Strictness
+from chromerag.extractor import ChromeRAG
+from chromerag.input_quality import InputQualityReport, assess_input_html
+from chromerag.models import ExtractResult
+from chromerag.site_chrome import (
+    SiteChromeModel,
+    load_chrome_models,
+    mine_site_chrome,
+    save_chrome_models,
+)
+
+__all__ = [
+    "ChromeRAG",
+    "ExtractResult",
+    "PipelineConfig",
+    "ContentPriority",
+    "Strictness",
+    "PageType",
+    "SiteChromeModel",
+    "mine_site_chrome",
+    "save_chrome_models",
+    "load_chrome_models",
+    "learn_then_extract",
+    "assess_input_html",
+    "InputQualityReport",
+]
+__version__ = "0.1.0"
+__paper__ = (
+    "ChromeRAG: Ingest-Time Elimination of Site Template Noise for Enterprise Web RAG"
+)