itops-rag 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. itops_rag-0.1.0/.gitignore +32 -0
  2. itops_rag-0.1.0/LICENSE +21 -0
  3. itops_rag-0.1.0/PKG-INFO +282 -0
  4. itops_rag-0.1.0/README.md +226 -0
  5. itops_rag-0.1.0/pyproject.toml +117 -0
  6. itops_rag-0.1.0/src/itops_rag/README.md +3 -0
  7. itops_rag-0.1.0/src/itops_rag/__init__.py +30 -0
  8. itops_rag-0.1.0/src/itops_rag/config.py +125 -0
  9. itops_rag-0.1.0/src/itops_rag/core/README.md +3 -0
  10. itops_rag-0.1.0/src/itops_rag/core/__init__.py +1 -0
  11. itops_rag-0.1.0/src/itops_rag/core/chunking.py +149 -0
  12. itops_rag-0.1.0/src/itops_rag/core/context.py +81 -0
  13. itops_rag-0.1.0/src/itops_rag/core/costs.py +98 -0
  14. itops_rag-0.1.0/src/itops_rag/core/embeddings.py +110 -0
  15. itops_rag-0.1.0/src/itops_rag/core/generate.py +246 -0
  16. itops_rag-0.1.0/src/itops_rag/core/index.py +486 -0
  17. itops_rag-0.1.0/src/itops_rag/core/models.py +192 -0
  18. itops_rag-0.1.0/src/itops_rag/core/redact.py +118 -0
  19. itops_rag-0.1.0/src/itops_rag/core/rerank.py +54 -0
  20. itops_rag-0.1.0/src/itops_rag/core/retriever.py +151 -0
  21. itops_rag-0.1.0/src/itops_rag/evaluation.py +257 -0
  22. itops_rag-0.1.0/src/itops_rag/integrations/README.md +3 -0
  23. itops_rag-0.1.0/src/itops_rag/integrations/__init__.py +1 -0
  24. itops_rag-0.1.0/src/itops_rag/integrations/langchain.py +40 -0
  25. itops_rag-0.1.0/src/itops_rag/interfaces/README.md +3 -0
  26. itops_rag-0.1.0/src/itops_rag/interfaces/__init__.py +1 -0
  27. itops_rag-0.1.0/src/itops_rag/interfaces/cli.py +316 -0
  28. itops_rag-0.1.0/src/itops_rag/interfaces/mcp_server.py +109 -0
  29. itops_rag-0.1.0/src/itops_rag/interfaces/slack_bot.py +69 -0
  30. itops_rag-0.1.0/src/itops_rag/interfaces/static/README.md +3 -0
  31. itops_rag-0.1.0/src/itops_rag/interfaces/static/index.html +146 -0
  32. itops_rag-0.1.0/src/itops_rag/interfaces/web.py +99 -0
  33. itops_rag-0.1.0/src/itops_rag/loaders/README.md +3 -0
  34. itops_rag-0.1.0/src/itops_rag/loaders/__init__.py +98 -0
  35. itops_rag-0.1.0/src/itops_rag/loaders/common.py +83 -0
  36. itops_rag-0.1.0/src/itops_rag/loaders/confluence.py +107 -0
  37. itops_rag-0.1.0/src/itops_rag/loaders/markdown.py +28 -0
  38. itops_rag-0.1.0/src/itops_rag/loaders/pdf.py +48 -0
  39. itops_rag-0.1.0/src/itops_rag/loaders/records.py +60 -0
  40. itops_rag-0.1.0/src/itops_rag/loaders/tickets.py +113 -0
  41. itops_rag-0.1.0/src/itops_rag/prompts/README.md +3 -0
  42. itops_rag-0.1.0/src/itops_rag/prompts/__init__.py +1 -0
  43. itops_rag-0.1.0/src/itops_rag/prompts/packs/README.md +3 -0
  44. itops_rag-0.1.0/src/itops_rag/prompts/packs/base/README.md +3 -0
  45. itops_rag-0.1.0/src/itops_rag/prompts/packs/base/qa.yaml +13 -0
  46. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/README.md +3 -0
  47. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/change_risk_review.yaml +21 -0
  48. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/incident_summary.yaml +16 -0
  49. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/kb_gap.yaml +16 -0
  50. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/onboarding_qa.yaml +14 -0
  51. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/oncall_handover.yaml +16 -0
  52. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/policy_check.yaml +16 -0
  53. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/postmortem_lessons.yaml +16 -0
  54. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/runbook_steps.yaml +16 -0
  55. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/similar_tickets.yaml +15 -0
  56. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/suggested_reply.yaml +19 -0
  57. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/ticket_triage.yaml +24 -0
  58. itops_rag-0.1.0/src/itops_rag/prompts/packs/itops/troubleshoot.yaml +19 -0
  59. itops_rag-0.1.0/src/itops_rag/prompts/packs/personal/README.md +3 -0
  60. itops_rag-0.1.0/src/itops_rag/prompts/packs/personal/decision_recall.yaml +14 -0
  61. itops_rag-0.1.0/src/itops_rag/prompts/packs/personal/notes_qa.yaml +14 -0
  62. itops_rag-0.1.0/src/itops_rag/prompts/registry.py +151 -0
  63. itops_rag-0.1.0/src/itops_rag/py.typed +0 -0
  64. itops_rag-0.1.0/src/itops_rag/stores/README.md +3 -0
  65. itops_rag-0.1.0/src/itops_rag/stores/__init__.py +1 -0
  66. itops_rag-0.1.0/src/itops_rag/stores/docstore.py +347 -0
  67. itops_rag-0.1.0/src/itops_rag/stores/vectors.py +194 -0
@@ -0,0 +1,32 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ .venv/
5
+ .mypy_cache/
6
+ .ruff_cache/
7
+ .pytest_cache/
8
+ .coverage
9
+ htmlcov/
10
+ dist/
11
+ build/
12
+ *.egg-info/
13
+
14
+ # Local state: indexes, cost ledger, model cache, secrets
15
+ .itops-rag/
16
+ .env
17
+ *.sqlite
18
+ *.sqlite-wal
19
+ *.sqlite-shm
20
+ .fastembed_cache/
21
+ local_cache/
22
+
23
+ # OS / editors
24
+ .DS_Store
25
+ .idea/
26
+ .vscode/
27
+
28
+ # Local editor/agent config
29
+ .claude/
30
+
31
+ # Connector exports can hold real company content
32
+ exports/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Chamudith Amarasinghe
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,282 @@
1
+ Metadata-Version: 2.5
2
+ Name: itops-rag
3
+ Version: 0.1.0
4
+ Summary: Retrieval-augmented answers for IT operations: hybrid search, local reranking, cited answers from Claude, an MCP server and a reusable RAG engine.
5
+ Project-URL: Homepage, https://github.com/chamu262/itops-rag
6
+ Project-URL: Issues, https://github.com/chamu262/itops-rag/issues
7
+ Author: Chamudith Amarasinghe
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: chromadb,claude,entra,intune,it-operations,jamf,mcp,rag,retrieval,sqlite-vec
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: System Administrators
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: System :: Systems Administration
20
+ Classifier: Typing :: Typed
21
+ Requires-Python: >=3.11
22
+ Requires-Dist: anthropic>=2.10
23
+ Requires-Dist: beautifulsoup4>=4.12
24
+ Requires-Dist: fastembed>=0.6
25
+ Requires-Dist: numpy>=1.26
26
+ Requires-Dist: pydantic-settings>=2.7
27
+ Requires-Dist: pydantic>=2.7
28
+ Requires-Dist: pypdf>=5.0
29
+ Requires-Dist: pyyaml>=6.0
30
+ Requires-Dist: rich>=13.7
31
+ Requires-Dist: sqlite-vec>=0.1.6
32
+ Requires-Dist: typer>=0.12
33
+ Provides-Extra: all
34
+ Requires-Dist: chromadb>=1.0; extra == 'all'
35
+ Requires-Dist: fastapi>=0.115; extra == 'all'
36
+ Requires-Dist: jinja2>=3.1; extra == 'all'
37
+ Requires-Dist: langchain-core>=0.3; extra == 'all'
38
+ Requires-Dist: mcp>=2.0; extra == 'all'
39
+ Requires-Dist: python-multipart>=0.0.9; extra == 'all'
40
+ Requires-Dist: slack-bolt>=1.20; extra == 'all'
41
+ Requires-Dist: uvicorn>=0.30; extra == 'all'
42
+ Provides-Extra: chroma
43
+ Requires-Dist: chromadb>=1.0; extra == 'chroma'
44
+ Provides-Extra: langchain
45
+ Requires-Dist: langchain-core>=0.3; extra == 'langchain'
46
+ Provides-Extra: mcp
47
+ Requires-Dist: mcp>=2.0; extra == 'mcp'
48
+ Provides-Extra: slack
49
+ Requires-Dist: slack-bolt>=1.20; extra == 'slack'
50
+ Provides-Extra: web
51
+ Requires-Dist: fastapi>=0.115; extra == 'web'
52
+ Requires-Dist: jinja2>=3.1; extra == 'web'
53
+ Requires-Dist: python-multipart>=0.0.9; extra == 'web'
54
+ Requires-Dist: uvicorn>=0.30; extra == 'web'
55
+ Description-Content-Type: text/markdown
56
+
57
+ # itops-rag
58
+
59
+ **Retrieval-augmented answers for IT operations, built the way an IT systems engineer would want it:** cited
60
+ answers from runbooks, KB articles, past tickets, change records, post-mortems and vendor docs; hybrid search
61
+ that finds error codes like `AADSTS53003` and `0x80180014`; local embeddings and reranking (no per-query cost);
62
+ "I don't know" instead of guesses; and an MCP server so Claude Code and other agents can use the same knowledge.
63
+
64
+ [![CI](https://github.com/chamu262/itops-rag/actions/workflows/ci.yml/badge.svg)](https://github.com/chamu262/itops-rag/actions/workflows/ci.yml)
65
+ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)
66
+ ![License](https://img.shields.io/badge/license-MIT-green)
67
+
68
+ > **Sanitized demo of private production work.** Everything in `data/demo/` describes a made-up company,
69
+ > *Kestrel Cloud* (fictional people, tenants, hostnames, tickets and a fictional VPN called MeshGate). The engine is the
70
+ > same one I run against real IT documentation; real-use configuration and live connectors live in a private repo.
71
+
72
+ ![Web UI: search for an Autopilot error code, showing redacted, access-tagged sources (a past ticket that matches exactly, then the runbook and vendor notes further down)](docs/img/web-search.jpg)
73
+
74
+ ## What it is for
75
+
76
+ | Who | Typical question | Prompt |
77
+ |---|---|---|
78
+ | On-call engineer | "MeshGate shows MG-503 for everyone in Lisbon, what do I check?" | `troubleshoot` |
79
+ | Sysadmin | "Steps to re-enrol a Mac that dropped out of Jamf" | `runbook_steps` |
80
+ | Help desk (L1) | Paste a ticket and get category, priority, team, KB to send and similar tickets | `ticket_triage` |
81
+ | Help desk (L1) | Draft a plain-language reply the agent can send | `suggested_reply` |
82
+ | CAB reviewer | Paste a change plan and get the risks from past changes and incidents | `change_risk_review` |
83
+ | Incident lead | Stakeholder summary of an outage from the post-mortem | `incident_summary` |
84
+ | Team lead | Recurring lessons and open actions across post-mortems | `postmortem_lessons` |
85
+ | New joiner | "How does SSO work between Entra and Google Workspace here?" | `onboarding_qa` |
86
+ | Anyone | "Can I share a Drive folder with a contractor?" (quotes the rule) | `policy_check` |
87
+ | AI agents | `search_kb`, `get_context`, `similar_tickets` over MCP | MCP server |
88
+
89
+ There are 12 IT ops prompts, a generic `qa` prompt and a `personal` pack. See [docs/prompts.md](docs/prompts.md).
90
+
91
+ ## Architecture
92
+
93
+ ```mermaid
94
+ flowchart LR
95
+ subgraph Sources
96
+ MD[Runbooks / KB<br/>Markdown]
97
+ PDF[Vendor docs<br/>PDF]
98
+ CF[Confluence<br/>HTML export]
99
+ TK[Tickets<br/>Jira CSV / ServiceNow JSON]
100
+ CH[Changes & post-mortems<br/>YAML / Markdown]
101
+ end
102
+
103
+ subgraph Ingest["Ingest (local, free)"]
104
+ L[Loaders<br/>+ metadata & access tags] --> R[Redaction<br/>secrets & PII]
105
+ R --> C[Heading-aware chunking<br/>800 / 100]
106
+ C --> E[Embeddings<br/>all-MiniLM-L6-v2 · ONNX]
107
+ end
108
+
109
+ subgraph Index["Index (one folder per app)"]
110
+ DS[(SQLite docstore<br/>+ FTS5 BM25)]
111
+ VS[(Vectors<br/>sqlite-vec or Chroma)]
112
+ end
113
+
114
+ subgraph Query
115
+ Q[Question + role] --> H[Hybrid search<br/>BM25 + vector → RRF]
116
+ H --> RR[Cross-encoder rerank<br/>ms-marco-MiniLM · local<br/>+ doc-type prior, 1 hit per doc]
117
+ RR --> G{Refusal gate<br/>score + error-code check}
118
+ G -- thin context --> IDK["I don't know"<br/>no API call]
119
+ G -- ok --> X[Small-to-big<br/>hit → section or whole doc<br/>~6k-token budget]
120
+ X --> P[Prompt template<br/>+ fenced &lt;context&gt;]
121
+ P --> LLM[Claude<br/>Opus 5.5 / Haiku 5.5]
122
+ LLM --> V[Citation check<br/>+ Haiku faithfulness check]
123
+ end
124
+
125
+ Sources --> L
126
+ E --> VS
127
+ C --> DS
128
+ DS --> H
129
+ VS --> H
130
+ V --> OUT[CLI · Web UI · MCP · Slack]
131
+ ```
132
+
133
+ More detail, including the design decisions and their trade-offs: [docs/architecture.md](docs/architecture.md).
134
+
135
+ ## Quick start
136
+
137
+ ```bash
138
+ git clone https://github.com/chamu262/itops-rag && cd itops-rag
139
+ python3.12 -m venv .venv && source .venv/bin/activate
140
+ pip install -e ".[web,mcp]"
141
+
142
+ itops-rag ingest data/demo/kestrel-cloud # ~10 s, models download once (~110 MB)
143
+ itops-rag search "MG-401 after macOS update" # free: local search, no API key
144
+ itops-rag ask "MG-401 after macOS update" --dry-run # free: shows the exact prompt Claude would get
145
+
146
+ cp .env.example .env # add your ANTHROPIC_API_KEY
147
+ itops-rag ask "MeshGate shows MG-503 for everyone in Lisbon" --prompt troubleshoot
148
+ itops-rag ask "Triage this" --prompt ticket_triage --input examples/ticket.txt
149
+ itops-rag ask "Review this change" --prompt change_risk_review --input examples/change-plan.txt
150
+ itops-rag serve # web UI on http://127.0.0.1:8080
151
+ ```
152
+
153
+ The web UI treats every caller as `public` unless your SSO proxy vouches for a higher role: set
154
+ `ITOPS_RAG_ROLE_HEADER_SECRET`, and have the proxy send `X-ITOPS-Role` plus `X-ITOPS-Proxy-Secret` (and strip any
155
+ copies the client sent). The CLI, library and MCP server use `ITOPS_RAG_DEFAULT_ROLE`.
156
+
157
+ ![Terminal demo: ingest, search, a local refusal and the eval (recorded from real runs by scripts/make_demo_gif.py)](docs/img/demo.gif)
158
+
159
+ When the index has nothing relevant, it says so without calling the model. Here the question names an error code
160
+ that no document mentions:
161
+
162
+ ![Web UI: a question about an unknown error code refused locally at no cost](docs/img/web-refusal.jpg)
163
+
164
+ Docker: `docker compose up --build`, then open http://localhost:8080. The image has the models baked in, so it runs
165
+ offline.
166
+
167
+ ## Use it from Claude Code (MCP)
168
+
169
+ ```bash
170
+ claude mcp add itops-rag \
171
+ -e ITOPS_RAG_INDEX_DIR="$PWD/.itops-rag/index" \
172
+ -e ITOPS_RAG_MODEL_CACHE_DIR="$HOME/.cache/itops-rag-models" \
173
+ -- "$PWD/.venv/bin/itops-rag-mcp"
174
+ ```
175
+
176
+ | Tool | Cost | What it returns |
177
+ |---|---|---|
178
+ | `search_kb` | free | Top chunks with doc id, section, scores, access tag |
179
+ | `get_context` | free | Retrieval and the fully rendered grounded prompt. **The calling agent answers with its own model, so there is no second API bill.** |
180
+ | `similar_tickets` | free | Past tickets like a symptom, with their resolutions |
181
+ | `get_document` | free | One full document by id |
182
+ | `list_prompts` | free | The prompt library |
183
+ | `answer_question` | tokens | Full Claude answer via the server's own key (off unless `ITOPS_RAG_MCP_ALLOW_ANSWERS=true`) |
184
+
185
+ All tools are annotated read-only. The caller's role is fixed when the server starts, never taken from a tool
186
+ argument. This fits alongside my [poc.mcp-it-tools](https://github.com/chamu262/poc.mcp-it-tools) server: that one
187
+ reads live systems, this one reads the documentation about them.
188
+
189
+ ## Use it as a library (one engine, many indexes)
190
+
191
+ ```python
192
+ from pathlib import Path
193
+ from itops_rag import Index, Settings
194
+
195
+ settings = Settings(index_dir=Path("~/.my-agent/rag").expanduser(), store="sqlite",
196
+ assistant_name="Nova", audience="one person's private notes", default_role="public")
197
+ with Index.open(settings) as ix:
198
+ ix.ingest_path(Path("~/notes").expanduser(), prune_missing=True)
199
+ hits = ix.search("which backup schedule did I pick?") # free
200
+ answer = ix.ask("When do my backups run?", prompt="notes_qa") # cited answer from Claude
201
+ ```
202
+
203
+ `pip install itops-rag` is the light base install (SQLite + ONNX, no torch, no server). Extras: `[chroma]`, `[web]`,
204
+ `[mcp]`, `[slack]`, `[langchain]`, `[all]`. Each app keeps its own index folder and prompt pack. The engine is shared;
205
+ the data never is. Full example: [examples/library_usage.py](examples/library_usage.py).
206
+
207
+ ## Retrieval quality
208
+
209
+ Measured offline (no API key) on the 65-question golden set ([eval/](eval/)): 54 answerable questions across all
210
+ three roles, including 16 deliberately hard paraphrases, plus 11 that should be refused, 5 of them near-misses
211
+ ("Okta SAML rotation", "MG-777"). Each question is searched exactly the way `ask()` searches it (its prompt
212
+ template's doc types and top-k), k = 4. **Primary** means the one document that should answer it (the runbook,
213
+ KB article or standard), not just any related ticket. Reproduce with `itops-rag eval`.
214
+
215
+ | Mode | hit@4 | MRR | primary hit@4 | primary MRR | recall@4 | ticket share | refusal accuracy | access leaks |
216
+ |---|---|---|---|---|---|---|---|---|
217
+ | vector | 0.94 | 0.84 | 0.93 | 0.77 | 0.70 | 0.17 | – | 0 |
218
+ | keyword (BM25) | 0.96 | 0.80 | 0.89 | 0.72 | 0.70 | 0.18 | – | 0 |
219
+ | hybrid (RRF) | 0.98 | 0.86 | 0.93 | 0.75 | 0.74 | 0.21 | – | 0 |
220
+ | **hybrid + rerank** | **0.98** | **0.91** | **0.98** | **0.85** | **0.75** | **0.10** | **0.95** | **0** |
221
+
222
+ What moved the numbers (the [review](docs/review.md) has the experiments, including the ones that didn't help):
223
+
224
+ - **Doc-type prior and one hit per document.** In this corpus 82 of 145 documents are tickets, and they used to
225
+ crowd runbooks out of the top 4. Adding +2 to the rerank score of authoritative doc types (for ordering only)
226
+ raised primary hit@4 from 0.93 to 0.98 and primary MRR from 0.80 to 0.85.
227
+ - **Hybrid beats either half.** Vector search misses error-code questions and keyword search misses paraphrases.
228
+ - **The free gate refuses 8 of 11 unanswerable questions**, including unknown error codes (the question names a code
229
+ no retrieved document mentions). The 3 near-misses it lets through score as high as real questions, so Claude's
230
+ own "I don't know" rule is the second line of defence.
231
+ - **Kept as is, after measuring alternatives:** the embedding model (bge-small, arctic-embed, nomic and bge-base were
232
+ no better after reranking) and the reranker (bge-reranker-base was slower and worse; ms-marco-L12 was twice the
233
+ latency).
234
+
235
+ Caveat: the questions were written knowing the corpus. Treat these as regression numbers, not a benchmark. CI fails
236
+ if primary hit, MRR or refusal accuracy drop (see `eval-gate` in the workflow).
237
+
238
+ ## Safety and cost controls
239
+
240
+ | Control | How |
241
+ |---|---|
242
+ | Citations required | Every claim cites `[n]`. Citations are checked against the chunks actually supplied; uncited or invalid citations raise warnings. |
243
+ | "I don't know" | If the best rerank score is below a threshold, it refuses **locally, with no API call**, and names the owning team of the nearest docs. |
244
+ | Retrieved text is data | Chunks are fenced in `<context>`, tag-like text is neutralised, injection patterns are flagged `suspicious="true"`, and the system prompt says to ignore and report them. Tested with planted attacks in `data/demo/kestrel-cloud/adversarial/`. |
245
+ | Redaction at ingest | AWS, GitHub, Slack, Anthropic, Azure and Google keys, `sk-`/`pk-` keys, JWTs, Bearer/Basic auth headers, private keys (even truncated ones), `name = value` secrets (`DB_PASSWORD`, `aws_secret_access_key`, quoted values), emails and phone numbers are masked before embedding, plus IPs in tickets. People fields are dropped from ticket exports. A test ingests the adversarial fixtures and asserts that no planted secret survives. |
246
+ | Access tags | `public` < `it-staff` < `security` per document, enforced inside both the keyword and the vector query. Documents with no tag default to `it-staff`, not `public`. The web UI is `public` unless a proxy-verified role header says otherwise. |
247
+ | Cost | Embeddings, search, rerank, refusals and eval are local and free. Haiku 5.5 runs the cheap steps (triage, policy checks, the faithfulness check, under $0.001), and every call is logged with its cost against a monthly budget cap that blocks calls. Unknown or fallback models are charged at the highest listed price, so the cap errs on the safe side. Prompt caching is wired but rarely pays here: the system prompt (~330 tokens) is under the 512-token minimum, and every question has different context. |
248
+ | Model safety | Server-side fallbacks (`fallbacks: "default"`) are on for Opus, so a safety-classifier decline is retried in the same call. Turn it off with `ITOPS_RAG_SERVER_FALLBACKS=false`. |
249
+
250
+ Known limits: regexes can't find personal names in free text (loaders drop known people fields instead), and a
251
+ determined attacker can phrase an injection the heuristics miss. The fencing and system rules still apply, but treat
252
+ indexed content as untrusted.
253
+
254
+ ## Repository layout
255
+
256
+ | Path | What's there |
257
+ |---|---|
258
+ | [`src/itops_rag/core/`](src/itops_rag/core/) | Engine: models, chunking, redaction, embeddings, rerank, retriever, context fencing, generation, costs, `Index` |
259
+ | [`src/itops_rag/stores/`](src/itops_rag/stores/) | SQLite docstore + FTS5, sqlite-vec / numpy / Chroma vector stores |
260
+ | [`src/itops_rag/loaders/`](src/itops_rag/loaders/) | Markdown, PDF, Confluence HTML, ticket CSV/JSON, YAML records |
261
+ | [`src/itops_rag/prompts/`](src/itops_rag/prompts/) | Grounding system prompt and YAML prompt packs |
262
+ | [`src/itops_rag/interfaces/`](src/itops_rag/interfaces/) | CLI, web UI + API, MCP server, Slack stub |
263
+ | [`src/itops_rag/integrations/`](src/itops_rag/integrations/) | LangChain retriever adapter |
264
+ | [`data/demo/`](data/demo/) | The fictional Kestrel Cloud corpus (145 documents) |
265
+ | [`eval/`](eval/) | Golden set and baseline |
266
+ | [`examples/`](examples/) | Original LangChain pipeline, library usage, MCP config, sample ticket and change plan |
267
+ | [`docs/`](docs/) | Architecture, prompt library, operations guide, decision records |
268
+ | [`deploy/`](deploy/) | Slack app manifest, Cloud Run notes |
269
+ | [`tests/`](tests/) | pytest suite (offline: no models, no API key) |
270
+
271
+ ## Development
272
+
273
+ ```bash
274
+ pip install -e ".[all]" --group dev
275
+ pytest # 76 tests, offline
276
+ ruff check . && ruff format --check . && mypy
277
+ itops-rag eval --baseline eval/baseline.json
278
+ ```
279
+
280
+ ## Licence
281
+
282
+ MIT. Built by Chamudith Amarasinghe. The fictional demo data may be reused freely.
@@ -0,0 +1,226 @@
1
+ # itops-rag
2
+
3
+ **Retrieval-augmented answers for IT operations, built the way an IT systems engineer would want it:** cited
4
+ answers from runbooks, KB articles, past tickets, change records, post-mortems and vendor docs; hybrid search
5
+ that finds error codes like `AADSTS53003` and `0x80180014`; local embeddings and reranking (no per-query cost);
6
+ "I don't know" instead of guesses; and an MCP server so Claude Code and other agents can use the same knowledge.
7
+
8
+ [![CI](https://github.com/chamu262/itops-rag/actions/workflows/ci.yml/badge.svg)](https://github.com/chamu262/itops-rag/actions/workflows/ci.yml)
9
+ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)
10
+ ![License](https://img.shields.io/badge/license-MIT-green)
11
+
12
+ > **Sanitized demo of private production work.** Everything in `data/demo/` describes a made-up company,
13
+ > *Kestrel Cloud* (fictional people, tenants, hostnames, tickets and a fictional VPN called MeshGate). The engine is the
14
+ > same one I run against real IT documentation; real-use configuration and live connectors live in a private repo.
15
+
16
+ ![Web UI: search for an Autopilot error code, showing redacted, access-tagged sources (a past ticket that matches exactly, then the runbook and vendor notes further down)](docs/img/web-search.jpg)
17
+
18
+ ## What it is for
19
+
20
+ | Who | Typical question | Prompt |
21
+ |---|---|---|
22
+ | On-call engineer | "MeshGate shows MG-503 for everyone in Lisbon, what do I check?" | `troubleshoot` |
23
+ | Sysadmin | "Steps to re-enrol a Mac that dropped out of Jamf" | `runbook_steps` |
24
+ | Help desk (L1) | Paste a ticket and get category, priority, team, KB to send and similar tickets | `ticket_triage` |
25
+ | Help desk (L1) | Draft a plain-language reply the agent can send | `suggested_reply` |
26
+ | CAB reviewer | Paste a change plan and get the risks from past changes and incidents | `change_risk_review` |
27
+ | Incident lead | Stakeholder summary of an outage from the post-mortem | `incident_summary` |
28
+ | Team lead | Recurring lessons and open actions across post-mortems | `postmortem_lessons` |
29
+ | New joiner | "How does SSO work between Entra and Google Workspace here?" | `onboarding_qa` |
30
+ | Anyone | "Can I share a Drive folder with a contractor?" (quotes the rule) | `policy_check` |
31
+ | AI agents | `search_kb`, `get_context`, `similar_tickets` over MCP | MCP server |
32
+
33
+ There are 12 IT ops prompts, a generic `qa` prompt and a `personal` pack. See [docs/prompts.md](docs/prompts.md).
34
+
35
+ ## Architecture
36
+
37
+ ```mermaid
38
+ flowchart LR
39
+ subgraph Sources
40
+ MD[Runbooks / KB<br/>Markdown]
41
+ PDF[Vendor docs<br/>PDF]
42
+ CF[Confluence<br/>HTML export]
43
+ TK[Tickets<br/>Jira CSV / ServiceNow JSON]
44
+ CH[Changes & post-mortems<br/>YAML / Markdown]
45
+ end
46
+
47
+ subgraph Ingest["Ingest (local, free)"]
48
+ L[Loaders<br/>+ metadata & access tags] --> R[Redaction<br/>secrets & PII]
49
+ R --> C[Heading-aware chunking<br/>800 / 100]
50
+ C --> E[Embeddings<br/>all-MiniLM-L6-v2 · ONNX]
51
+ end
52
+
53
+ subgraph Index["Index (one folder per app)"]
54
+ DS[(SQLite docstore<br/>+ FTS5 BM25)]
55
+ VS[(Vectors<br/>sqlite-vec or Chroma)]
56
+ end
57
+
58
+ subgraph Query
59
+ Q[Question + role] --> H[Hybrid search<br/>BM25 + vector → RRF]
60
+ H --> RR[Cross-encoder rerank<br/>ms-marco-MiniLM · local<br/>+ doc-type prior, 1 hit per doc]
61
+ RR --> G{Refusal gate<br/>score + error-code check}
62
+ G -- thin context --> IDK["I don't know"<br/>no API call]
63
+ G -- ok --> X[Small-to-big<br/>hit → section or whole doc<br/>~6k-token budget]
64
+ X --> P[Prompt template<br/>+ fenced &lt;context&gt;]
65
+ P --> LLM[Claude<br/>Opus 5.5 / Haiku 5.5]
66
+ LLM --> V[Citation check<br/>+ Haiku faithfulness check]
67
+ end
68
+
69
+ Sources --> L
70
+ E --> VS
71
+ C --> DS
72
+ DS --> H
73
+ VS --> H
74
+ V --> OUT[CLI · Web UI · MCP · Slack]
75
+ ```
76
+
77
+ More detail, including the design decisions and their trade-offs: [docs/architecture.md](docs/architecture.md).
78
+
79
+ ## Quick start
80
+
81
+ ```bash
82
+ git clone https://github.com/chamu262/itops-rag && cd itops-rag
83
+ python3.12 -m venv .venv && source .venv/bin/activate
84
+ pip install -e ".[web,mcp]"
85
+
86
+ itops-rag ingest data/demo/kestrel-cloud # ~10 s, models download once (~110 MB)
87
+ itops-rag search "MG-401 after macOS update" # free: local search, no API key
88
+ itops-rag ask "MG-401 after macOS update" --dry-run # free: shows the exact prompt Claude would get
89
+
90
+ cp .env.example .env # add your ANTHROPIC_API_KEY
91
+ itops-rag ask "MeshGate shows MG-503 for everyone in Lisbon" --prompt troubleshoot
92
+ itops-rag ask "Triage this" --prompt ticket_triage --input examples/ticket.txt
93
+ itops-rag ask "Review this change" --prompt change_risk_review --input examples/change-plan.txt
94
+ itops-rag serve # web UI on http://127.0.0.1:8080
95
+ ```
96
+
97
+ The web UI treats every caller as `public` unless your SSO proxy vouches for a higher role: set
98
+ `ITOPS_RAG_ROLE_HEADER_SECRET`, and have the proxy send `X-ITOPS-Role` plus `X-ITOPS-Proxy-Secret` (and strip any
99
+ copies the client sent). The CLI, library and MCP server use `ITOPS_RAG_DEFAULT_ROLE`.
100
+
101
+ ![Terminal demo: ingest, search, a local refusal and the eval (recorded from real runs by scripts/make_demo_gif.py)](docs/img/demo.gif)
102
+
103
+ When the index has nothing relevant, it says so without calling the model. Here the question names an error code
104
+ that no document mentions:
105
+
106
+ ![Web UI: a question about an unknown error code refused locally at no cost](docs/img/web-refusal.jpg)
107
+
108
+ Docker: `docker compose up --build`, then open http://localhost:8080. The image has the models baked in, so it runs
109
+ offline.
110
+
111
+ ## Use it from Claude Code (MCP)
112
+
113
+ ```bash
114
+ claude mcp add itops-rag \
115
+ -e ITOPS_RAG_INDEX_DIR="$PWD/.itops-rag/index" \
116
+ -e ITOPS_RAG_MODEL_CACHE_DIR="$HOME/.cache/itops-rag-models" \
117
+ -- "$PWD/.venv/bin/itops-rag-mcp"
118
+ ```
119
+
120
+ | Tool | Cost | What it returns |
121
+ |---|---|---|
122
+ | `search_kb` | free | Top chunks with doc id, section, scores, access tag |
123
+ | `get_context` | free | Retrieval and the fully rendered grounded prompt. **The calling agent answers with its own model, so there is no second API bill.** |
124
+ | `similar_tickets` | free | Past tickets like a symptom, with their resolutions |
125
+ | `get_document` | free | One full document by id |
126
+ | `list_prompts` | free | The prompt library |
127
+ | `answer_question` | tokens | Full Claude answer via the server's own key (off unless `ITOPS_RAG_MCP_ALLOW_ANSWERS=true`) |
128
+
129
+ All tools are annotated read-only. The caller's role is fixed when the server starts, never taken from a tool
130
+ argument. This fits alongside my [poc.mcp-it-tools](https://github.com/chamu262/poc.mcp-it-tools) server: that one
131
+ reads live systems, this one reads the documentation about them.
132
+
133
+ ## Use it as a library (one engine, many indexes)
134
+
135
+ ```python
136
+ from pathlib import Path
137
+ from itops_rag import Index, Settings
138
+
139
+ settings = Settings(index_dir=Path("~/.my-agent/rag").expanduser(), store="sqlite",
140
+ assistant_name="Nova", audience="one person's private notes", default_role="public")
141
+ with Index.open(settings) as ix:
142
+ ix.ingest_path(Path("~/notes").expanduser(), prune_missing=True)
143
+ hits = ix.search("which backup schedule did I pick?") # free
144
+ answer = ix.ask("When do my backups run?", prompt="notes_qa") # cited answer from Claude
145
+ ```
146
+
147
+ `pip install itops-rag` is the light base install (SQLite + ONNX, no torch, no server). Extras: `[chroma]`, `[web]`,
148
+ `[mcp]`, `[slack]`, `[langchain]`, `[all]`. Each app keeps its own index folder and prompt pack. The engine is shared;
149
+ the data never is. Full example: [examples/library_usage.py](examples/library_usage.py).
150
+
151
+ ## Retrieval quality
152
+
153
+ Measured offline (no API key) on the 65-question golden set ([eval/](eval/)): 54 answerable questions across all
154
+ three roles, including 16 deliberately hard paraphrases, plus 11 that should be refused, 5 of them near-misses
155
+ ("Okta SAML rotation", "MG-777"). Each question is searched exactly the way `ask()` searches it (its prompt
156
+ template's doc types and top-k), k = 4. **Primary** means the one document that should answer it (the runbook,
157
+ KB article or standard), not just any related ticket. Reproduce with `itops-rag eval`.
158
+
159
+ | Mode | hit@4 | MRR | primary hit@4 | primary MRR | recall@4 | ticket share | refusal accuracy | access leaks |
160
+ |---|---|---|---|---|---|---|---|---|
161
+ | vector | 0.94 | 0.84 | 0.93 | 0.77 | 0.70 | 0.17 | – | 0 |
162
+ | keyword (BM25) | 0.96 | 0.80 | 0.89 | 0.72 | 0.70 | 0.18 | – | 0 |
163
+ | hybrid (RRF) | 0.98 | 0.86 | 0.93 | 0.75 | 0.74 | 0.21 | – | 0 |
164
+ | **hybrid + rerank** | **0.98** | **0.91** | **0.98** | **0.85** | **0.75** | **0.10** | **0.95** | **0** |
165
+
166
+ What moved the numbers (the [review](docs/review.md) has the experiments, including the ones that didn't help):
167
+
168
+ - **Doc-type prior and one hit per document.** In this corpus 82 of 145 documents are tickets, and they used to
169
+ crowd runbooks out of the top 4. Adding +2 to the rerank score of authoritative doc types (for ordering only)
170
+ raised primary hit@4 from 0.93 to 0.98 and primary MRR from 0.80 to 0.85.
171
+ - **Hybrid beats either half.** Vector search misses error-code questions and keyword search misses paraphrases.
172
+ - **The free gate refuses 8 of 11 unanswerable questions**, including unknown error codes (the question names a code
173
+ no retrieved document mentions). The 3 near-misses it lets through score as high as real questions, so Claude's
174
+ own "I don't know" rule is the second line of defence.
175
+ - **Kept as is, after measuring alternatives:** the embedding model (bge-small, arctic-embed, nomic and bge-base were
176
+ no better after reranking) and the reranker (bge-reranker-base was slower and worse; ms-marco-L12 was twice the
177
+ latency).
178
+
179
+ Caveat: the questions were written knowing the corpus. Treat these as regression numbers, not a benchmark. CI fails
180
+ if primary hit, MRR or refusal accuracy drop (see `eval-gate` in the workflow).
181
+
182
+ ## Safety and cost controls
183
+
184
+ | Control | How |
185
+ |---|---|
186
+ | Citations required | Every claim cites `[n]`. Citations are checked against the chunks actually supplied; uncited or invalid citations raise warnings. |
187
+ | "I don't know" | If the best rerank score is below a threshold, it refuses **locally, with no API call**, and names the owning team of the nearest docs. |
188
+ | Retrieved text is data | Chunks are fenced in `<context>`, tag-like text is neutralised, injection patterns are flagged `suspicious="true"`, and the system prompt says to ignore and report them. Tested with planted attacks in `data/demo/kestrel-cloud/adversarial/`. |
189
+ | Redaction at ingest | AWS, GitHub, Slack, Anthropic, Azure and Google keys, `sk-`/`pk-` keys, JWTs, Bearer/Basic auth headers, private keys (even truncated ones), `name = value` secrets (`DB_PASSWORD`, `aws_secret_access_key`, quoted values), emails and phone numbers are masked before embedding, plus IPs in tickets. People fields are dropped from ticket exports. A test ingests the adversarial fixtures and asserts that no planted secret survives. |
190
+ | Access tags | `public` < `it-staff` < `security` per document, enforced inside both the keyword and the vector query. Documents with no tag default to `it-staff`, not `public`. The web UI is `public` unless a proxy-verified role header says otherwise. |
191
+ | Cost | Embeddings, search, rerank, refusals and eval are local and free. Haiku 5.5 runs the cheap steps (triage, policy checks, the faithfulness check, under $0.001), and every call is logged with its cost against a monthly budget cap that blocks calls. Unknown or fallback models are charged at the highest listed price, so the cap errs on the safe side. Prompt caching is wired but rarely pays here: the system prompt (~330 tokens) is under the 512-token minimum, and every question has different context. |
192
+ | Model safety | Server-side fallbacks (`fallbacks: "default"`) are on for Opus, so a safety-classifier decline is retried in the same call. Turn it off with `ITOPS_RAG_SERVER_FALLBACKS=false`. |
193
+
194
+ Known limits: regexes can't find personal names in free text (loaders drop known people fields instead), and a
195
+ determined attacker can phrase an injection the heuristics miss. The fencing and system rules still apply, but treat
196
+ indexed content as untrusted.
197
+
198
+ ## Repository layout
199
+
200
+ | Path | What's there |
201
+ |---|---|
202
+ | [`src/itops_rag/core/`](src/itops_rag/core/) | Engine: models, chunking, redaction, embeddings, rerank, retriever, context fencing, generation, costs, `Index` |
203
+ | [`src/itops_rag/stores/`](src/itops_rag/stores/) | SQLite docstore + FTS5, sqlite-vec / numpy / Chroma vector stores |
204
+ | [`src/itops_rag/loaders/`](src/itops_rag/loaders/) | Markdown, PDF, Confluence HTML, ticket CSV/JSON, YAML records |
205
+ | [`src/itops_rag/prompts/`](src/itops_rag/prompts/) | Grounding system prompt and YAML prompt packs |
206
+ | [`src/itops_rag/interfaces/`](src/itops_rag/interfaces/) | CLI, web UI + API, MCP server, Slack stub |
207
+ | [`src/itops_rag/integrations/`](src/itops_rag/integrations/) | LangChain retriever adapter |
208
+ | [`data/demo/`](data/demo/) | The fictional Kestrel Cloud corpus (145 documents) |
209
+ | [`eval/`](eval/) | Golden set and baseline |
210
+ | [`examples/`](examples/) | Original LangChain pipeline, library usage, MCP config, sample ticket and change plan |
211
+ | [`docs/`](docs/) | Architecture, prompt library, operations guide, decision records |
212
+ | [`deploy/`](deploy/) | Slack app manifest, Cloud Run notes |
213
+ | [`tests/`](tests/) | pytest suite (offline: no models, no API key) |
214
+
215
+ ## Development
216
+
217
+ ```bash
218
+ pip install -e ".[all]" --group dev
219
+ pytest # 76 tests, offline
220
+ ruff check . && ruff format --check . && mypy
221
+ itops-rag eval --baseline eval/baseline.json
222
+ ```
223
+
224
+ ## Licence
225
+
226
+ MIT. Built by Chamudith Amarasinghe. The fictional demo data may be reused freely.