gsas-query 0.2.0__tar.gz

gsas_query-0.2.0/.env.example

@@ -0,0 +1,29 @@
+# ── LLM Backend ──────────────────────────────────────────────────────────────
+# "ollama"    — fully local, free, no data leaves the network (RECOMMENDED)
+# "anthropic" — Anthropic Claude API (requires ANTHROPIC_API_KEY below)
+LLM_BACKEND=ollama
+
+# ── Ollama settings (used when LLM_BACKEND=ollama) ───────────────────────────
+# Install Ollama: https://ollama.com
+# Pull a model:  ollama pull llama3
+# Recommended models (balance of quality and speed):
+#   llama3          — 8B params, good quality, ~8 GB RAM
+#   llama3:70b      — 70B params, excellent quality, ~48 GB RAM / GPU
+#   mistral         — fast, ~5 GB RAM
+#   phi3            — lightweight, ~4 GB RAM
+OLLAMA_URL=http://localhost:11434
+OLLAMA_MODEL=llama3
+
+# ── Anthropic API (only needed if LLM_BACKEND=anthropic) ─────────────────────
+# ANTHROPIC_API_KEY=sk-ant-...
+# ANTHROPIC_MODEL=claude-sonnet-4-6
+
+# ── Server settings ───────────────────────────────────────────────────────────
+# Comma-separated origins allowed for CORS. Use "*" to allow all.
+ALLOWED_ORIGINS=*
+
+# Requests per minute per IP (0 to disable)
+RATE_LIMIT_RPM=30
+
+# Secret key required for POST /ingest (leave blank to disable auth on that route)
+ADMIN_KEY=

gsas_query-0.2.0/.github/workflows/ingest.yml

@@ -0,0 +1,72 @@
+name: Build ChromaDB index
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "gsas_query/sources.py"
+      - "gsas_query/ingest.py"
+      - ".github/workflows/ingest.yml"
+  workflow_dispatch:
+    inputs:
+      include_pdfs:
+        description: "Include PDF ingestion (slower, needs more RAM)"
+        type: boolean
+        default: false
+
+jobs:
+  ingest:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-ingest-${{ hashFiles('pyproject.toml') }}
+
+      # CPU-only torch is ~400 MB; the default CUDA wheel is ~2.5 GB and
+      # causes OOM (exit 143) on GitHub-hosted runners.
+      - name: Install CPU-only PyTorch
+        run: pip install torch --index-url https://download.pytorch.org/whl/cpu
+
+      - name: Install dependencies
+        run: pip install chromadb sentence-transformers beautifulsoup4 requests pypdf lxml
+
+      - name: Install gsas_query package
+        run: pip install -e .
+
+      - name: Build ChromaDB index (HTML only by default)
+        env:
+          TOKENIZERS_PARALLELISM: "false"
+          OMP_NUM_THREADS: "1"
+          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
+        run: python ingest.py --html-only
+
+      - name: Show index stats
+        env:
+          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
+        run: |
+          python -c "
+          import chromadb, os
+          path = os.path.join(os.environ['GSAS_QUERY_DATA_DIR'], 'chroma_db')
+          c = chromadb.PersistentClient(path=path)
+          col = c.get_collection('gsasii_docs')
+          print(f'Chunks in index: {col.count()}')
+          "
+
+      - name: Upload chroma_db artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: chroma_db
+          path: ci_data/chroma_db/
+          retention-days: 90

gsas_query-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read   # required for actions/checkout
+      id-token: write  # required for OIDC trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

gsas_query-0.2.0/.gitignore

@@ -0,0 +1,23 @@
+# Environment
+.env
+*.env.local
+
+# Vector store (rebuild with ingest.py — do not commit)
+chroma_db/
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+.venv/
+venv/
+env/
+*.egg-info/
+dist/
+build/
+
+# macOS
+.DS_Store
+
+# Downloaded PDFs (re-fetched during ingestion)
+*.pdf

gsas_query-0.2.0/PKG-INFO

@@ -0,0 +1,283 @@
+Metadata-Version: 2.4
+Name: gsas-query
+Version: 0.2.0
+Summary: GSAS-II Documentation Assistant — semantic search and AI answers over GSAS-II tutorials and help pages
+Project-URL: Repository, https://github.com/pawantr/Query-GSAS
+Author: Pawan Tripathi
+License: MIT
+Keywords: crystallography,documentation,gsas,rag,xrd
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: chromadb>=0.5.0
+Requires-Dist: fastapi>=0.110.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pypdf>=4.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: uvicorn[standard]>=0.29.0
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.25.0; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: gui
+Requires-Dist: wxpython>=4.2.0; extra == 'gui'
+Description-Content-Type: text/markdown
+
+# gsas-query — GSAS-II Documentation Assistant
+
+Semantic search + AI answers over the full GSAS-II documentation set:
+**129 HTML pages** (home, help, and all 62 tutorials) plus the
+**Programmer's Guide** and **Powder Crystallography** book PDFs.
+
+Answers include **inline citations** — every `[N]` in the response is a
+clickable link to the exact documentation section that supported that sentence.
+
+All embedding and retrieval runs on your machine — no data is sent externally
+unless you explicitly choose the Anthropic API backend.
+
+---
+
+## Installation
+
+### pip
+
+```bash
+pip install gsas-query
+```
+
+Or install directly from source:
+
+```bash
+pip install git+https://github.com/pawantr/Query-GSAS.git
+```
+
+### conda
+
+```bash
+conda install -c conda-forge gsas-query
+```
+
+### Into an existing GSAS-II environment
+
+```bash
+# Activate the GSAS-II conda environment first, then:
+pip install gsas-query
+```
+
+### Development / editable install
+
+```bash
+git clone https://github.com/pawantr/Query-GSAS.git
+cd Query-GSAS
+pip install -e ".[dev]"
+```
+
+---
+
+## Ollama — free local LLM (recommended)
+
+```bash
+brew install ollama          # macOS; see https://ollama.com for other platforms
+ollama serve &               # start the local server
+ollama pull llama3           # ~5 GB one-time download
+```
+
+Other model options: `llama3:70b` (better quality, ~40 GB), `mistral` (faster, ~4 GB).
+
+---
+
+## First-time setup — index the documentation
+
+Run once, or again when the docs are updated. Fetches ~130 web pages and 2 PDFs,
+embeds everything locally. Takes ~10–20 minutes.
+
+The index is stored in `~/.gsas_query/chroma_db` (override with `GSAS_QUERY_DATA_DIR`).
+
+```bash
+gsas-query --setup              # all sources (HTML + PDFs)
+gsas-query --setup --html-only  # skip PDFs, faster (~5 min)
+gsas-query --setup --reset      # drop index and rebuild from scratch
+```
+
+---
+
+## Usage
+
+### Command-line — single question
+
+```bash
+gsas-query "How do I set up a sequential refinement?"
+gsas-query "What parameters control the background in Rietveld?"
+gsas-query "How do I export a CIF for publication?"
+```
+
+### Command-line — interactive REPL
+
+Multi-turn conversation that remembers previous questions in the session.
+
+```bash
+gsas-query
+```
+
+```
+GSAS-II Documentation Assistant
+════════════════════════════════════════════════════════════════════════════════
+
+Knowledge base: 3,847 indexed chunks.
+LLM backend: ollama
+Type your question and press Enter. 'clear' resets history, 'quit' exits.
+
+────────────────────────────────────────────────────────────────────────────────
+You: How do I constrain lattice parameters?
+Thinking…
+Assistant: To constrain lattice parameters in GSAS-II, open the Constraints
+tab in the Phase panel [1]…
+
+Sources:
+  [94%] Help: Phase General  ›  Constraints
+         https://advancedphotonsource.github.io/GSAS-II-tutorials/help/phasegeneral.html
+```
+
+Commands inside the REPL: `clear` (reset history), `quit` / `exit` (exit).
+
+### Desktop GUI
+
+Opens a standalone floating dialog — stays open while you work in GSAS-II.
+
+```bash
+gsas-query --gui
+```
+
+### Embed in GSAS-II Help menu
+
+Add one call to the GSAS-II menu handler (e.g. in `GSASIIctrl.py`):
+
+```python
+def OnDocAssistant(self, event):
+    try:
+        from gsas_query.gui import show_assistant
+        show_assistant(self)          # self = GSAS-II main frame
+    except ImportError:
+        wx.MessageBox(
+            "GSAS-II Assistant not installed.\n"
+            "Run: pip install gsas-query",
+            "Not available"
+        )
+```
+
+`show_assistant()` is idempotent — calling it a second time raises the existing
+window rather than opening a duplicate.
+
+### Web UI
+
+```bash
+gsas-query-web                          # serves on 0.0.0.0:8000
+HOST=127.0.0.1 PORT=8765 gsas-query-web
+```
+
+Then open `http://localhost:8000` in a browser. The web UI shows inline citations
+as clickable superscript links and source chips below each answer.
+
+---
+
+## CLI flags reference
+
+| Flag | Description |
+|---|---|
+| `--setup` | Index all documentation sources |
+| `--setup --reset` | Drop the existing index and rebuild |
+| `--setup --html-only` | Index HTML only, skip PDFs |
+| `--gui` | Open the wxPython desktop assistant |
+| `--backend ollama\|anthropic\|retrieval` | Override `LLM_BACKEND` env var |
+| `--model <name>` | Override Ollama or Anthropic model |
+| `--stats` | Show chunk count, backend, and DB path |
+
+---
+
+## LLM backend configuration
+
+Set `LLM_BACKEND` in a `.env` file or the environment:
+
+| Backend | Config | Notes |
+|---|---|---|
+| `ollama` (default) | `OLLAMA_MODEL=llama3` | Free, fully local — no data leaves the network |
+| `anthropic` | `ANTHROPIC_API_KEY=sk-ant-…` | Better answers; queries sent to Anthropic |
+| `retrieval` | — | No LLM — returns raw matched chunks; useful offline or for testing |
+
+```bash
+gsas-query --backend retrieval "What is Le Bail extraction?"
+gsas-query --backend ollama --model mistral "How do I index peaks?"
+```
+
+---
+
+## Inline citations
+
+When using Ollama or Anthropic backends, answers contain `[N]` markers inline.
+In the **web UI** these render as clickable superscript links opening the exact
+source section. In the **CLI**, source URLs are listed below the answer with
+relevance scores.
+
+---
+
+## Doc sources
+
+| Category | Count |
+|---|---|
+| Home / installation pages | 22 |
+| Help pages (all sections) | 42 |
+| Tutorials | 62 |
+| Programmer's Guide (PDF, readthedocs) | 1 |
+| Powder Crystallography book (PDF, auto-fetches latest release) | 1 |
+| **Total** | **128 sources** |
+
+All HTML sources are fetched from
+`https://advancedphotonsource.github.io/GSAS-II-tutorials/`.
+The book PDF is fetched from the latest GitHub release of
+[briantoby/PowderCrystallography](https://github.com/briantoby/PowderCrystallography/releases).
+
+---
+
+## Re-indexing when docs update
+
+```bash
+gsas-query --setup --reset
+```
+
+Or trigger via the web API (requires `ADMIN_KEY` set in `.env`):
+
+```bash
+curl -X POST http://localhost:8000/ingest -H "X-Admin-Key: your-key"
+```
+
+---
+
+## Security and deployment notes
+
+- All embeddings and vector search run locally (sentence-transformers, ChromaDB).
+- Ollama runs entirely on-premises — no queries leave the network.
+- The `anthropic` backend sends question text and retrieved doc chunks to Anthropic.
+  Do not use it in air-gapped or data-sensitive environments.
+- The web server applies per-IP rate limiting (default 30 req/min, configurable
+  via `RATE_LIMIT_RPM` in `.env`).
+- The `/ingest` endpoint is protected by `X-Admin-Key`; leave `ADMIN_KEY` blank
+  to disable remote re-indexing.
+
+---
+
+## User data directory
+
+The ChromaDB index is stored at `~/.gsas_query/chroma_db` by default.
+Override with the `GSAS_QUERY_DATA_DIR` environment variable:
+
+```bash
+GSAS_QUERY_DATA_DIR=/data/gsas_query gsas-query --setup
+```

gsas_query-0.2.0/README.md

@@ -0,0 +1,250 @@
+# gsas-query — GSAS-II Documentation Assistant
+
+Semantic search + AI answers over the full GSAS-II documentation set:
+**129 HTML pages** (home, help, and all 62 tutorials) plus the
+**Programmer's Guide** and **Powder Crystallography** book PDFs.
+
+Answers include **inline citations** — every `[N]` in the response is a
+clickable link to the exact documentation section that supported that sentence.
+
+All embedding and retrieval runs on your machine — no data is sent externally
+unless you explicitly choose the Anthropic API backend.
+
+---
+
+## Installation
+
+### pip
+
+```bash
+pip install gsas-query
+```
+
+Or install directly from source:
+
+```bash
+pip install git+https://github.com/pawantr/Query-GSAS.git
+```
+
+### conda
+
+```bash
+conda install -c conda-forge gsas-query
+```
+
+### Into an existing GSAS-II environment
+
+```bash
+# Activate the GSAS-II conda environment first, then:
+pip install gsas-query
+```
+
+### Development / editable install
+
+```bash
+git clone https://github.com/pawantr/Query-GSAS.git
+cd Query-GSAS
+pip install -e ".[dev]"
+```
+
+---
+
+## Ollama — free local LLM (recommended)
+
+```bash
+brew install ollama          # macOS; see https://ollama.com for other platforms
+ollama serve &               # start the local server
+ollama pull llama3           # ~5 GB one-time download
+```
+
+Other model options: `llama3:70b` (better quality, ~40 GB), `mistral` (faster, ~4 GB).
+
+---
+
+## First-time setup — index the documentation
+
+Run once, or again when the docs are updated. Fetches ~130 web pages and 2 PDFs,
+embeds everything locally. Takes ~10–20 minutes.
+
+The index is stored in `~/.gsas_query/chroma_db` (override with `GSAS_QUERY_DATA_DIR`).
+
+```bash
+gsas-query --setup              # all sources (HTML + PDFs)
+gsas-query --setup --html-only  # skip PDFs, faster (~5 min)
+gsas-query --setup --reset      # drop index and rebuild from scratch
+```
+
+---
+
+## Usage
+
+### Command-line — single question
+
+```bash
+gsas-query "How do I set up a sequential refinement?"
+gsas-query "What parameters control the background in Rietveld?"
+gsas-query "How do I export a CIF for publication?"
+```
+
+### Command-line — interactive REPL
+
+Multi-turn conversation that remembers previous questions in the session.
+
+```bash
+gsas-query
+```
+
+```
+GSAS-II Documentation Assistant
+════════════════════════════════════════════════════════════════════════════════
+
+Knowledge base: 3,847 indexed chunks.
+LLM backend: ollama
+Type your question and press Enter. 'clear' resets history, 'quit' exits.
+
+────────────────────────────────────────────────────────────────────────────────
+You: How do I constrain lattice parameters?
+Thinking…
+Assistant: To constrain lattice parameters in GSAS-II, open the Constraints
+tab in the Phase panel [1]…
+
+Sources:
+  [94%] Help: Phase General  ›  Constraints
+         https://advancedphotonsource.github.io/GSAS-II-tutorials/help/phasegeneral.html
+```
+
+Commands inside the REPL: `clear` (reset history), `quit` / `exit` (exit).
+
+### Desktop GUI
+
+Opens a standalone floating dialog — stays open while you work in GSAS-II.
+
+```bash
+gsas-query --gui
+```
+
+### Embed in GSAS-II Help menu
+
+Add one call to the GSAS-II menu handler (e.g. in `GSASIIctrl.py`):
+
+```python
+def OnDocAssistant(self, event):
+    try:
+        from gsas_query.gui import show_assistant
+        show_assistant(self)          # self = GSAS-II main frame
+    except ImportError:
+        wx.MessageBox(
+            "GSAS-II Assistant not installed.\n"
+            "Run: pip install gsas-query",
+            "Not available"
+        )
+```
+
+`show_assistant()` is idempotent — calling it a second time raises the existing
+window rather than opening a duplicate.
+
+### Web UI
+
+```bash
+gsas-query-web                          # serves on 0.0.0.0:8000
+HOST=127.0.0.1 PORT=8765 gsas-query-web
+```
+
+Then open `http://localhost:8000` in a browser. The web UI shows inline citations
+as clickable superscript links and source chips below each answer.
+
+---
+
+## CLI flags reference
+
+| Flag | Description |
+|---|---|
+| `--setup` | Index all documentation sources |
+| `--setup --reset` | Drop the existing index and rebuild |
+| `--setup --html-only` | Index HTML only, skip PDFs |
+| `--gui` | Open the wxPython desktop assistant |
+| `--backend ollama\|anthropic\|retrieval` | Override `LLM_BACKEND` env var |
+| `--model <name>` | Override Ollama or Anthropic model |
+| `--stats` | Show chunk count, backend, and DB path |
+
+---
+
+## LLM backend configuration
+
+Set `LLM_BACKEND` in a `.env` file or the environment:
+
+| Backend | Config | Notes |
+|---|---|---|
+| `ollama` (default) | `OLLAMA_MODEL=llama3` | Free, fully local — no data leaves the network |
+| `anthropic` | `ANTHROPIC_API_KEY=sk-ant-…` | Better answers; queries sent to Anthropic |
+| `retrieval` | — | No LLM — returns raw matched chunks; useful offline or for testing |
+
+```bash
+gsas-query --backend retrieval "What is Le Bail extraction?"
+gsas-query --backend ollama --model mistral "How do I index peaks?"
+```
+
+---
+
+## Inline citations
+
+When using Ollama or Anthropic backends, answers contain `[N]` markers inline.
+In the **web UI** these render as clickable superscript links opening the exact
+source section. In the **CLI**, source URLs are listed below the answer with
+relevance scores.
+
+---
+
+## Doc sources
+
+| Category | Count |
+|---|---|
+| Home / installation pages | 22 |
+| Help pages (all sections) | 42 |
+| Tutorials | 62 |
+| Programmer's Guide (PDF, readthedocs) | 1 |
+| Powder Crystallography book (PDF, auto-fetches latest release) | 1 |
+| **Total** | **128 sources** |
+
+All HTML sources are fetched from
+`https://advancedphotonsource.github.io/GSAS-II-tutorials/`.
+The book PDF is fetched from the latest GitHub release of
+[briantoby/PowderCrystallography](https://github.com/briantoby/PowderCrystallography/releases).
+
+---
+
+## Re-indexing when docs update
+
+```bash
+gsas-query --setup --reset
+```
+
+Or trigger via the web API (requires `ADMIN_KEY` set in `.env`):
+
+```bash
+curl -X POST http://localhost:8000/ingest -H "X-Admin-Key: your-key"
+```
+
+---
+
+## Security and deployment notes
+
+- All embeddings and vector search run locally (sentence-transformers, ChromaDB).
+- Ollama runs entirely on-premises — no queries leave the network.
+- The `anthropic` backend sends question text and retrieved doc chunks to Anthropic.
+  Do not use it in air-gapped or data-sensitive environments.
+- The web server applies per-IP rate limiting (default 30 req/min, configurable
+  via `RATE_LIMIT_RPM` in `.env`).
+- The `/ingest` endpoint is protected by `X-Admin-Key`; leave `ADMIN_KEY` blank
+  to disable remote re-indexing.
+
+---
+
+## User data directory
+
+The ChromaDB index is stored at `~/.gsas_query/chroma_db` by default.
+Override with the `GSAS_QUERY_DATA_DIR` environment variable:
+
+```bash
+GSAS_QUERY_DATA_DIR=/data/gsas_query gsas-query --setup
+```

gsas_query-0.2.0/app.py

@@ -0,0 +1,4 @@
+# Backwards-compatible stub — the app object is now in gsas_query.app.
+# Start the server with: gsas-query-web
+# or: uvicorn gsas_query.app:app
+from gsas_query.app import app  # noqa: F401

gsas_query-0.2.0/cli.py

@@ -0,0 +1,6 @@
+# Backwards-compatible stub — delegates to the gsas_query package.
+# Run `gsas-query` instead if installed via pip.
+from gsas_query.cli import main
+
+if __name__ == "__main__":
+    main()