gsas2-query 0.3.0__tar.gz

gsas2_query-0.3.0/.env.example

@@ -0,0 +1,45 @@
+# ── LLM Backend ──────────────────────────────────────────────────────────────
+# Backend selection precedence:
+#   1. LLM_BACKEND env var (if set) always takes effect.
+#   2. If LLM_BACKEND is NOT set and llama-cpp-python is installed, llama_cpp
+#      is selected automatically.
+#   3. If neither of the above applies, "ollama" is the default.
+#
+# "ollama"    — fully local, free, no data leaves the network
+# "llama_cpp" — in-process llama-cpp-python (auto-selected if installed)
+# "anthropic" — Anthropic Claude API (requires ANTHROPIC_API_KEY below)
+# "retrieval" — no LLM; returns raw matched chunks (offline / testing)
+LLM_BACKEND=ollama
+
+# ── llama-cpp-python settings (used when llama_cpp backend is active) ─────────
+# Install: conda install -c conda-forge llama-cpp-python
+#          or: pip install "gsas-query[llama_cpp]"
+# Download a GGUF model from Hugging Face, then set the path below.
+# LLAMA_CPP_MODEL=/path/to/model.gguf
+# LLAMA_CPP_N_CTX=4096       # context window size (tokens)
+# LLAMA_CPP_MAX_TOKENS=1500  # maximum tokens to generate
+
+# ── Ollama settings (used when LLM_BACKEND=ollama) ───────────────────────────
+# Install Ollama: https://ollama.com
+# Pull a model:  ollama pull llama3
+# Recommended models (balance of quality and speed):
+#   llama3          — 8B params, good quality, ~8 GB RAM
+#   llama3:70b      — 70B params, excellent quality, ~48 GB RAM / GPU
+#   mistral         — fast, ~5 GB RAM
+#   phi3            — lightweight, ~4 GB RAM
+OLLAMA_URL=http://localhost:11434
+OLLAMA_MODEL=llama3
+
+# ── Anthropic API (only needed if LLM_BACKEND=anthropic) ─────────────────────
+# ANTHROPIC_API_KEY=sk-ant-...
+# ANTHROPIC_MODEL=claude-sonnet-4-6
+
+# ── Server settings ───────────────────────────────────────────────────────────
+# Comma-separated origins allowed for CORS. Use "*" to allow all.
+ALLOWED_ORIGINS=*
+
+# Requests per minute per IP (0 to disable)
+RATE_LIMIT_RPM=30
+
+# Secret key required for POST /ingest (leave blank to disable auth on that route)
+ADMIN_KEY=

gsas2_query-0.3.0/.github/workflows/ingest.yml

@@ -0,0 +1,95 @@
+name: Build ChromaDB index
+# This creates a index to the GSAS-II documents and places it as
+# downloadable release file under the name 
+# https://github.com/<owner>/<repo>/releases/download/latest-chroma-db/chroma_db_latest.zip
+# (as well as a artifact)
+
+on:
+  workflow_dispatch:
+    inputs:
+      include_book:
+        description: "Include Book HTML contents"
+        type: boolean
+        default: true
+      include_manual:
+        description: "Include Programmers' Manual"
+        type: boolean
+        default: true
+
+jobs:
+  ingest:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      #contents: read
+      contents: write   # changed: needed to create/update release/tag
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-ingest-${{ hashFiles('pyproject.toml') }}
+
+      # Embeddings use chromadb's built-in ONNX runtime (all-MiniLM-L6-v2)
+      # so PyTorch and sentence-transformers are not required.
+      - name: Install gsas_query package
+        run: pip install -e ".[dev]"
+
+      - name: Build ChromaDB index (HTML only, contents by options)
+        env:
+          TOKENIZERS_PARALLELISM: "false"
+          OMP_NUM_THREADS: "1"
+          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
+        run: |
+          cmd="python ingest.py --html-only"
+          if [ "${{ inputs.include_book }}" = "true" ]; then
+            cmd="$cmd --book"
+          fi
+          if [ "${{ inputs.include_manual }}" = "true" ]; then
+            cmd="$cmd --manual"
+          fi
+          echo "Running: $cmd"
+          eval "$cmd"
+          
+      - name: Show index stats
+        env:
+          GSAS_QUERY_DATA_DIR: ${{ github.workspace }}/ci_data
+        run: |
+          python -c "
+          import chromadb, os
+          path = os.path.join(os.environ['GSAS_QUERY_DATA_DIR'], 'chroma_db')
+          c = chromadb.PersistentClient(path=path)
+          col = c.get_collection('gsasii_docs')
+          print(f'Chunks in index: {col.count()}')
+          "
+
+      - name: Upload chroma_db artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: chroma_db
+          path: ci_data/chroma_db/
+          retention-days: 90
+
+      - name: Package chroma_db as zip
+        run: |
+          cd ci_data
+          zip -r ../chroma_db_latest.zip chroma_db
+
+      - name: Create/Update stable release and upload asset
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: latest-chroma-db
+          name: Latest ChromaDB Index
+          body: Auto-updated ChromaDB index artifact from ingest workflow.
+          files: chroma_db_latest.zip
+          overwrite_files: true
+          make_latest: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

gsas2_query-0.3.0/.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read   # required for actions/checkout
+      id-token: write  # required for OIDC trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

gsas2_query-0.3.0/.gitignore

@@ -0,0 +1,27 @@
+# Environment
+.env
+*.env.local
+
+# Vector store (rebuild with ingest.py — do not commit)
+chroma_db/
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+.venv/
+venv/
+env/
+*.egg-info/
+dist/
+build/
+
+# macOS
+.DS_Store
+
+# Downloaded PDFs (re-fetched during ingestion)
+*.pdf
+manuscript/
+
+# Internal admin paperwork (disclosure drafts etc.)
+release/

gsas2_query-0.3.0/PKG-INFO

@@ -0,0 +1,333 @@
+Metadata-Version: 2.5
+Name: gsas2-query
+Version: 0.3.0
+Summary: Query-GSAS-II Documentation Assistant — semantic search and AI answers over GSAS-II tutorials and help pages
+Project-URL: Repository, https://github.com/AdvancedPhotonSource/Query-GSAS-II
+Author-email: Pawan Tripathi <ptripathi@anl.gov>, Brian Toby <toby@anl.gov>
+License: MIT
+Keywords: crystallography,documentation,gsas,rag,xrd
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Chemistry
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: chromadb>=0.5.0
+Requires-Dist: fastapi>=0.110.0
+Requires-Dist: fastembed>=0.3.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: lxml>=5.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pypdf>=4.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: uvicorn[standard]>=0.29.0
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.25.0; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: gui
+Requires-Dist: wxpython>=4.2.0; extra == 'gui'
+Provides-Extra: llama-cpp
+Requires-Dist: llama-cpp-python>=0.2.0; extra == 'llama-cpp'
+Description-Content-Type: text/markdown
+
+# gsas2-query — GSAS-II Documentation Assistant
+
+Semantic search + AI answers over the full GSAS-II documentation set:
+**129 HTML pages** (home, help, and all 62 tutorials) plus the
+**Programmer's Guide** and **Powder Crystallography** book PDFs.
+
+Answers include **inline citations** — every `[N]` in the response is a
+clickable link to the exact documentation section that supported that sentence.
+
+All embedding and retrieval runs on your machine — no data is sent externally
+unless you explicitly choose the Anthropic API backend.
+
+---
+
+## Installation
+
+### pip
+
+```bash
+pip install gsas2-query
+```
+
+Or install directly from source:
+
+```bash
+pip install git+https://github.com/AdvancedPhotonSource/Query-GSAS-II.git
+```
+
+### conda
+
+```bash
+conda install -c conda-forge gsas2-query
+```
+
+### Into an existing GSAS-II environment
+
+```bash
+# Activate the GSAS-II conda environment first, then:
+pip install gsas2-query
+```
+
+### Development / editable install
+
+```bash
+git clone https://github.com/AdvancedPhotonSource/Query-GSAS-II.git
+cd Query-GSAS-II
+pip install -e ".[dev]"
+```
+
+---
+
+## Ollama — free local LLM (recommended when llama.cpp is not installed)
+
+```bash
+brew install ollama          # macOS; see https://ollama.com for other platforms
+ollama serve &               # start the local server
+ollama pull llama3           # ~5 GB one-time download
+```
+
+Other model options: `llama3:70b` (better quality, ~40 GB), `mistral` (faster, ~4 GB).
+
+---
+
+## llama-cpp-python — in-process local LLM (auto-selected when installed)
+
+`llama-cpp-python` runs a GGUF model directly inside the Python process — no
+separate server required. It is cross-platform and available from conda-forge.
+
+When `llama-cpp-python` is importable, gsas2-query selects it **automatically**
+without any `LLM_BACKEND` setting.
+
+### Install
+
+```bash
+conda install -c conda-forge llama-cpp-python   # recommended
+# or:
+pip install "gsas2-query[llama_cpp]"
+```
+
+### Download a GGUF model
+
+Download any GGUF-format model from Hugging Face, for example:
+
+```bash
+# Llama 3.1 8B (Q4_K_M quantisation, ~5 GB):
+wget https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
+```
+
+### Configure
+
+Set the model path in your `.env` or environment:
+
+```env
+LLAMA_CPP_MODEL=/path/to/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
+# Optional tuning:
+# LLAMA_CPP_N_CTX=4096       # context window (default: 4096)
+# LLAMA_CPP_MAX_TOKENS=1500  # max tokens to generate (default: 1500)
+```
+
+---
+
+## First-time setup — index the documentation
+
+Run once, or again when the docs are updated. Fetches ~130 web pages and 2 PDFs,
+embeds everything locally. Takes ~10–20 minutes.
+
+By default the index is stored in `~/.GSASII/gsas_query/chroma_db`
+(see [User data directory](#userdatadirectory) for details).
+
+```bash
+gsas2-query --setup              # all sources (HTML + PDFs)
+gsas2-query --setup --html-only  # skip PDFs, faster (~5 min)
+gsas2-query --setup --reset      # drop index and rebuild from scratch
+```
+
+HTML-only index is 21 Mb. With GSAS-II Programmer's Guide, 41 Mb; with
+textbook as well, 53 Mb.
+
+---
+
+## Usage
+
+### Command-line — single question
+
+```bash
+gsas2-query "How do I set up a sequential refinement?"
+gsas2-query "What parameters control the background in Rietveld?"
+gsas2-query "How do I export a CIF for publication?"
+```
+
+### Command-line — interactive REPL
+
+Multi-turn conversation that remembers previous questions in the session.
+
+```bash
+gsas2-query
+```
+
+```
+GSAS-II Documentation Assistant
+════════════════════════════════════════════════════════════════════════════════
+
+Knowledge base: 3,847 indexed chunks.
+LLM backend: ollama
+Type your question and press Enter. 'clear' resets history, 'quit' exits.
+
+────────────────────────────────────────────────────────────────────────────────
+You: How do I constrain lattice parameters?
+Thinking…
+Assistant: To constrain lattice parameters in GSAS-II, open the Constraints
+tab in the Phase panel [1]…
+
+Sources:
+  [94%] Help: Phase General  ›  Constraints
+         https://advancedphotonsource.github.io/GSAS-II-tutorials/help/phasegeneral.html
+```
+
+Commands inside the REPL: `clear` (reset history), `quit` / `exit` (exit).
+
+### Desktop GUI
+
+Opens a standalone floating dialog — stays open while you work in GSAS-II.
+
+```bash
+gsas2-query --gui
+```
+
+### Embed in GSAS-II Help menu
+
+Add one call to the GSAS-II menu handler (e.g. in `GSASIIctrl.py`):
+
+```python
+def OnDocAssistant(self, event):
+    try:
+        from gsas_query.gui import show_assistant
+        show_assistant(self)          # self = GSAS-II main frame
+    except ImportError:
+        wx.MessageBox(
+            "GSAS-II Assistant not installed.\n"
+            "Run: pip install gsas2-query",
+            "Not available"
+        )
+```
+
+`show_assistant()` is idempotent — calling it a second time raises the existing
+window rather than opening a duplicate.
+
+### Web UI
+
+```bash
+gsas2-query-web                          # serves on 0.0.0.0:8000
+HOST=127.0.0.1 PORT=8765 gsas2-query-web
+```
+
+Then open `http://localhost:8000` in a browser. The web UI shows inline citations
+as clickable superscript links and source chips below each answer.
+
+---
+
+## CLI flags reference
+
+| Flag | Description |
+|---|---|
+| `--setup` | Index all documentation sources |
+| `--setup --reset` | Drop the existing index and rebuild |
+| `--setup --html-only` | Index HTML only, skip PDFs |
+| `--gui` | Open the wxPython desktop assistant |
+| `--backend ollama\|anthropic\|retrieval\|llama_cpp` | Override `LLM_BACKEND` env var |
+| `--model <name>` | Override OLLAMA_MODEL, ANTHROPIC_MODEL, or LLAMA_CPP_MODEL |
+| `--stats` | Show chunk count, backend, and DB path |
+
+---
+
+## LLM backend configuration
+
+Backend selection precedence:
+
+1. `LLM_BACKEND` env var (or `--backend` CLI flag) **always takes effect when set**.
+2. If `LLM_BACKEND` is **not set** and `llama-cpp-python` is importable, `llama_cpp` is selected automatically.
+3. Otherwise the default is `ollama`.
+
+| Backend | Config | Notes |
+|---|---|---|
+| `llama_cpp` (auto) | `LLAMA_CPP_MODEL=/path/to/model.gguf` | In-process, no daemon needed; auto-selected when llama-cpp-python is installed |
+| `ollama` (default) | `OLLAMA_MODEL=llama3` | Free, fully local — no data leaves the network |
+| `anthropic` | `ANTHROPIC_API_KEY=sk-ant-…` | Better answers; queries sent to Anthropic |
+| `retrieval` | — | No LLM — returns raw matched chunks; useful offline or for testing |
+
+```bash
+gsas2-query --backend retrieval "What is Le Bail extraction?"
+gsas2-query --backend ollama --model mistral "How do I index peaks?"
+gsas2-query --backend llama_cpp --model /path/to/model.gguf "How do I refine a structure?"
+```
+
+---
+
+## Inline citations
+
+When using Ollama, llama_cpp, or Anthropic backends, answers contain `[N]` markers inline.
+In the **web UI** these render as clickable superscript links opening the exact
+source section. In the **CLI**, source URLs are listed below the answer with
+relevance scores.
+
+---
+
+## Doc sources
+
+| Category | Count |
+|---|---|
+| Home / installation pages | 22 |
+| Help pages (all sections) | 42 |
+| Tutorials | 62 |
+| Programmer's Guide (PDF, readthedocs) | 1 |
+| Powder Crystallography book (PDF, auto-fetches latest release) | 1 |
+| **Total** | **128 sources** |
+
+All HTML sources are fetched from
+`https://advancedphotonsource.github.io/GSAS-II-tutorials/`.
+The book PDF is fetched from the latest GitHub release of
+[briantoby/PowderCrystallography](https://github.com/briantoby/PowderCrystallography/releases).
+
+---
+
+## Re-indexing when docs update
+
+```bash
+gsas2-query --setup --reset
+```
+
+Or trigger via the web API (requires `ADMIN_KEY` set in `.env`):
+
+```bash
+curl -X POST http://localhost:8000/ingest -H "X-Admin-Key: your-key"
+```
+
+---
+
+## Security and deployment notes
+
+- All embeddings and vector search run locally (sentence-transformers, ChromaDB).
+- Ollama runs entirely on-premises — no queries leave the network.
+- The `anthropic` backend sends question text and retrieved doc chunks to Anthropic.
+  Do not use it in air-gapped or data-sensitive environments.
+- The web server applies per-IP rate limiting (default 30 req/min, configurable
+  via `RATE_LIMIT_RPM` in `.env`).
+- The `/ingest` endpoint is protected by `X-Admin-Key`; leave `ADMIN_KEY` blank
+  to disable remote re-indexing.
+
+---
+
+## User data directory
+
+The ChromaDB index is stored at `~/.GSASII/gsas_query/chroma_db` by default.
+Override with the `GSAS_QUERY_DATA_DIR` environment variable:
+
+```bash
+GSAS_QUERY_DATA_DIR=/data/gsas_query gsas2-query --setup
+```