itops-rag 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- itops_rag/README.md +3 -0
- itops_rag/__init__.py +30 -0
- itops_rag/config.py +125 -0
- itops_rag/core/README.md +3 -0
- itops_rag/core/__init__.py +1 -0
- itops_rag/core/chunking.py +149 -0
- itops_rag/core/context.py +81 -0
- itops_rag/core/costs.py +98 -0
- itops_rag/core/embeddings.py +110 -0
- itops_rag/core/generate.py +246 -0
- itops_rag/core/index.py +486 -0
- itops_rag/core/models.py +192 -0
- itops_rag/core/redact.py +118 -0
- itops_rag/core/rerank.py +54 -0
- itops_rag/core/retriever.py +151 -0
- itops_rag/evaluation.py +257 -0
- itops_rag/integrations/README.md +3 -0
- itops_rag/integrations/__init__.py +1 -0
- itops_rag/integrations/langchain.py +40 -0
- itops_rag/interfaces/README.md +3 -0
- itops_rag/interfaces/__init__.py +1 -0
- itops_rag/interfaces/cli.py +316 -0
- itops_rag/interfaces/mcp_server.py +109 -0
- itops_rag/interfaces/slack_bot.py +69 -0
- itops_rag/interfaces/static/README.md +3 -0
- itops_rag/interfaces/static/index.html +146 -0
- itops_rag/interfaces/web.py +99 -0
- itops_rag/loaders/README.md +3 -0
- itops_rag/loaders/__init__.py +98 -0
- itops_rag/loaders/common.py +83 -0
- itops_rag/loaders/confluence.py +107 -0
- itops_rag/loaders/markdown.py +28 -0
- itops_rag/loaders/pdf.py +48 -0
- itops_rag/loaders/records.py +60 -0
- itops_rag/loaders/tickets.py +113 -0
- itops_rag/prompts/README.md +3 -0
- itops_rag/prompts/__init__.py +1 -0
- itops_rag/prompts/packs/README.md +3 -0
- itops_rag/prompts/packs/base/README.md +3 -0
- itops_rag/prompts/packs/base/qa.yaml +13 -0
- itops_rag/prompts/packs/itops/README.md +3 -0
- itops_rag/prompts/packs/itops/change_risk_review.yaml +21 -0
- itops_rag/prompts/packs/itops/incident_summary.yaml +16 -0
- itops_rag/prompts/packs/itops/kb_gap.yaml +16 -0
- itops_rag/prompts/packs/itops/onboarding_qa.yaml +14 -0
- itops_rag/prompts/packs/itops/oncall_handover.yaml +16 -0
- itops_rag/prompts/packs/itops/policy_check.yaml +16 -0
- itops_rag/prompts/packs/itops/postmortem_lessons.yaml +16 -0
- itops_rag/prompts/packs/itops/runbook_steps.yaml +16 -0
- itops_rag/prompts/packs/itops/similar_tickets.yaml +15 -0
- itops_rag/prompts/packs/itops/suggested_reply.yaml +19 -0
- itops_rag/prompts/packs/itops/ticket_triage.yaml +24 -0
- itops_rag/prompts/packs/itops/troubleshoot.yaml +19 -0
- itops_rag/prompts/packs/personal/README.md +3 -0
- itops_rag/prompts/packs/personal/decision_recall.yaml +14 -0
- itops_rag/prompts/packs/personal/notes_qa.yaml +14 -0
- itops_rag/prompts/registry.py +151 -0
- itops_rag/py.typed +0 -0
- itops_rag/stores/README.md +3 -0
- itops_rag/stores/__init__.py +1 -0
- itops_rag/stores/docstore.py +347 -0
- itops_rag/stores/vectors.py +194 -0
- itops_rag-0.1.0.dist-info/METADATA +282 -0
- itops_rag-0.1.0.dist-info/RECORD +67 -0
- itops_rag-0.1.0.dist-info/WHEEL +4 -0
- itops_rag-0.1.0.dist-info/entry_points.txt +3 -0
- itops_rag-0.1.0.dist-info/licenses/LICENSE +21 -0
itops_rag/README.md
ADDED
itops_rag/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""itops-rag: retrieval-augmented answers for IT operations, and a small reusable RAG engine.
|
|
2
|
+
|
|
3
|
+
Public API (stable within a minor version):
|
|
4
|
+
|
|
5
|
+
Index, Settings, Document, Chunk, Hit, Answer, PromptRegistry, PromptTemplate, visible_access
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from itops_rag.config import Settings
|
|
9
|
+
from itops_rag.core.index import Index, IndexMismatchError, IngestReport
|
|
10
|
+
from itops_rag.core.models import ACCESS_LEVELS, Answer, Chunk, Document, Hit, Usage, visible_access
|
|
11
|
+
from itops_rag.prompts.registry import PromptRegistry, PromptTemplate
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"ACCESS_LEVELS",
|
|
17
|
+
"Answer",
|
|
18
|
+
"Chunk",
|
|
19
|
+
"Document",
|
|
20
|
+
"Hit",
|
|
21
|
+
"Index",
|
|
22
|
+
"IndexMismatchError",
|
|
23
|
+
"IngestReport",
|
|
24
|
+
"PromptRegistry",
|
|
25
|
+
"PromptTemplate",
|
|
26
|
+
"Settings",
|
|
27
|
+
"Usage",
|
|
28
|
+
"__version__",
|
|
29
|
+
"visible_access",
|
|
30
|
+
]
|
itops_rag/config.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Settings, read from environment variables (prefix ``ITOPS_RAG_``), a ``.env`` file, or passed in code.
|
|
2
|
+
|
|
3
|
+
Every knob has a safe default, so ``Settings()`` works with no configuration at all.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Any, Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import Field, field_validator
|
|
12
|
+
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
|
13
|
+
|
|
14
|
+
StoreKind = Literal["auto", "sqlite", "chroma"]
|
|
15
|
+
SearchMode = Literal["vector", "keyword", "hybrid", "hybrid_rerank"]
|
|
16
|
+
Effort = Literal["low", "medium", "high", "xhigh", "max"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Settings(BaseSettings):
|
|
20
|
+
model_config = SettingsConfigDict(env_prefix="ITOPS_RAG_", env_file=".env", extra="ignore")
|
|
21
|
+
|
|
22
|
+
# Where the index lives. One folder per index: docstore.sqlite (+ chroma/ when that store is used).
|
|
23
|
+
index_dir: Path = Field(default=Path(".itops-rag/index"))
|
|
24
|
+
store: StoreKind = "auto"
|
|
25
|
+
"""``auto`` uses Chroma when it is installed, else SQLite (sqlite-vec, or numpy when the extension can't load)."""
|
|
26
|
+
|
|
27
|
+
# Local models (downloaded once to the fastembed cache, then fully offline).
|
|
28
|
+
embed_model: str = "sentence-transformers/all-MiniLM-L6-v2"
|
|
29
|
+
rerank_model: str = "Xenova/ms-marco-MiniLM-L-6-v2"
|
|
30
|
+
model_cache_dir: Path | None = None
|
|
31
|
+
|
|
32
|
+
# Chunking
|
|
33
|
+
chunk_size: int = 800
|
|
34
|
+
chunk_overlap: int = 100
|
|
35
|
+
|
|
36
|
+
# Retrieval
|
|
37
|
+
mode: SearchMode = "hybrid_rerank"
|
|
38
|
+
top_k: int = 4
|
|
39
|
+
candidates: int = 30
|
|
40
|
+
"""How many chunks each retriever returns before fusion, and how many go to the reranker."""
|
|
41
|
+
keyword_weight: float = 1.0
|
|
42
|
+
vector_weight: float = 1.0
|
|
43
|
+
max_chunks_per_doc: int = 1
|
|
44
|
+
"""Diversity cap: one hit per document; context expansion then brings in the rest of the section."""
|
|
45
|
+
doc_type_boost: dict[str, float] = Field(
|
|
46
|
+
default_factory=lambda: {
|
|
47
|
+
t: 2.0 for t in ("runbook", "kb", "vendor", "standard", "oncall", "postmortem", "change", "doc")
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
"""Added to rerank scores for ordering. Tickets get 0 so authoritative docs win ties with near-duplicate tickets.
|
|
51
|
+
Templates can override per type with their own ``doc_type_boost``. Measured: primary-doc hit 0.81 -> 0.94
|
|
52
|
+
on the hard golden questions."""
|
|
53
|
+
context_char_budget: int = 24000
|
|
54
|
+
"""About 6k tokens of context per answer, after expanding hits to their section or whole document."""
|
|
55
|
+
expand_whole_doc_chars: int = 4000
|
|
56
|
+
"""Documents shorter than this are given to the model whole; longer ones by section."""
|
|
57
|
+
min_rerank_score: float = -2.0
|
|
58
|
+
"""Refuse locally (no API call) when the best cross-encoder score is below this. Tuned with ``itops-rag eval``."""
|
|
59
|
+
min_vector_similarity: float = 0.30
|
|
60
|
+
"""Refusal gate used when reranking is off."""
|
|
61
|
+
|
|
62
|
+
# Identity used in the system prompt (a personal agent would set its own name and audience).
|
|
63
|
+
assistant_name: str = "IT Ops RAG"
|
|
64
|
+
audience: str = "the IT operations team"
|
|
65
|
+
|
|
66
|
+
# Access
|
|
67
|
+
default_role: Literal["public", "it-staff", "security"] = "it-staff"
|
|
68
|
+
"""Role for the CLI, the library and the MCP server (fixed per process)."""
|
|
69
|
+
default_access: Literal["public", "it-staff", "security"] = "it-staff"
|
|
70
|
+
"""Access tag for documents that don't declare one. Fail closed: not ``public``."""
|
|
71
|
+
web_default_role: Literal["public", "it-staff", "security"] = "public"
|
|
72
|
+
"""Role for web requests that don't carry a verified role header."""
|
|
73
|
+
role_header_secret: str = ""
|
|
74
|
+
"""If set, the web app trusts ``X-ITOPS-Role`` only when ``X-ITOPS-Proxy-Secret`` matches (your SSO proxy sets
|
|
75
|
+
both and strips any client-sent copies). If empty, the role header is ignored entirely."""
|
|
76
|
+
|
|
77
|
+
# Generation (Claude)
|
|
78
|
+
answer_model: str = "claude-opus-5-5"
|
|
79
|
+
fast_model: str = "claude-haiku-5-5"
|
|
80
|
+
"""Used for cheap steps: query rewriting, ticket triage labels, the faithfulness check."""
|
|
81
|
+
effort: Effort = "medium"
|
|
82
|
+
max_tokens: int = 8192
|
|
83
|
+
"""Includes thinking tokens (Opus 5.5 always thinks); you pay only for what is generated."""
|
|
84
|
+
server_fallbacks: bool = True
|
|
85
|
+
"""Send ``fallbacks: "default"`` so a safety-classifier decline is retried on a fallback model in the same call."""
|
|
86
|
+
rewrite_queries: bool = False
|
|
87
|
+
verify_answers: bool = True
|
|
88
|
+
"""A Haiku faithfulness check per answer (under $0.001). Off with ITOPS_RAG_VERIFY_ANSWERS=false."""
|
|
89
|
+
|
|
90
|
+
# Cost control
|
|
91
|
+
monthly_budget_usd: float = 25.0
|
|
92
|
+
cost_ledger: Path = Field(default=Path(".itops-rag/costs.jsonl"))
|
|
93
|
+
|
|
94
|
+
# Redaction at ingest
|
|
95
|
+
redact: bool = True
|
|
96
|
+
redact_keep_emails: Annotated[list[str], NoDecode] = Field(
|
|
97
|
+
default_factory=lambda: [r"^(servicedesk|it-oncall|security|noreply)@kestrelcloud\.example$"]
|
|
98
|
+
)
|
|
99
|
+
"""Regexes for addresses that are functional mailboxes, not people. They are kept."""
|
|
100
|
+
|
|
101
|
+
# Prompt packs
|
|
102
|
+
prompt_dirs: Annotated[list[Path], NoDecode] = Field(default_factory=list)
|
|
103
|
+
"""Extra folders of prompt YAML files, loaded after the built-in packs (same id overrides)."""
|
|
104
|
+
|
|
105
|
+
@field_validator("prompt_dirs", "redact_keep_emails", mode="before")
|
|
106
|
+
@classmethod
|
|
107
|
+
def _split_list(cls, value: Any) -> Any:
|
|
108
|
+
"""Accept a JSON list, or a comma-separated string, from environment variables."""
|
|
109
|
+
if isinstance(value, str):
|
|
110
|
+
text = value.strip()
|
|
111
|
+
if text.startswith("["):
|
|
112
|
+
import json
|
|
113
|
+
|
|
114
|
+
return json.loads(text)
|
|
115
|
+
return [part.strip() for part in text.split(",") if part.strip()]
|
|
116
|
+
return value
|
|
117
|
+
|
|
118
|
+
def resolved_store(self) -> Literal["sqlite", "chroma"]:
|
|
119
|
+
if self.store != "auto":
|
|
120
|
+
return self.store
|
|
121
|
+
try:
|
|
122
|
+
import chromadb # noqa: F401
|
|
123
|
+
except ImportError:
|
|
124
|
+
return "sqlite"
|
|
125
|
+
return "chroma"
|
itops_rag/core/README.md
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""See README.md in this folder."""
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Heading-aware chunking.
|
|
2
|
+
|
|
3
|
+
Markdown is first cut into sections at headings (so a chunk never spans two procedures), then each section is split
|
|
4
|
+
recursively on paragraph, line, sentence and word boundaries to at most ``size`` characters with ``overlap``
|
|
5
|
+
characters carried over. Each chunk keeps its heading breadcrumb, which is prepended when embedding.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from itops_rag.core.models import Chunk, Document
|
|
14
|
+
|
|
15
|
+
CHUNKER_VERSION = 2
|
|
16
|
+
"""Bump when splitting or ``Chunk.embed_text`` changes, so ingest re-chunks instead of skipping unchanged docs."""
|
|
17
|
+
|
|
18
|
+
_HEADING = re.compile(r"^(#{1,6})\s+(.+?)\s*#*\s*$")
|
|
19
|
+
_SEPARATORS = ("\n\n", "\n", ". ", "; ", ", ", " ")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class Section:
|
|
24
|
+
heading: str
|
|
25
|
+
text: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def split_sections(text: str) -> list[Section]:
|
|
29
|
+
"""Split Markdown into sections keyed by their heading breadcrumb (``H2 > H3``). Code fences are respected."""
|
|
30
|
+
sections: list[Section] = []
|
|
31
|
+
stack: list[tuple[int, str]] = []
|
|
32
|
+
buf: list[str] = []
|
|
33
|
+
in_fence = False
|
|
34
|
+
|
|
35
|
+
def flush() -> None:
|
|
36
|
+
body = "\n".join(buf).strip()
|
|
37
|
+
if body:
|
|
38
|
+
sections.append(Section(" > ".join(h for _, h in stack), body))
|
|
39
|
+
buf.clear()
|
|
40
|
+
|
|
41
|
+
for line in text.splitlines():
|
|
42
|
+
if line.lstrip().startswith(("```", "~~~")):
|
|
43
|
+
in_fence = not in_fence
|
|
44
|
+
m = None if in_fence else _HEADING.match(line)
|
|
45
|
+
if m:
|
|
46
|
+
flush()
|
|
47
|
+
level = len(m.group(1))
|
|
48
|
+
# The H1 is normally the document title, which is already on every chunk.
|
|
49
|
+
if level == 1:
|
|
50
|
+
stack.clear()
|
|
51
|
+
continue
|
|
52
|
+
while stack and stack[-1][0] >= level:
|
|
53
|
+
stack.pop()
|
|
54
|
+
stack.append((level, m.group(2).strip()))
|
|
55
|
+
else:
|
|
56
|
+
buf.append(line)
|
|
57
|
+
flush()
|
|
58
|
+
return sections
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def split_text(text: str, size: int, overlap: int) -> list[str]:
|
|
62
|
+
"""Recursive character splitter (same idea as LangChain's, no dependency)."""
|
|
63
|
+
if size <= 0:
|
|
64
|
+
raise ValueError("size must be positive")
|
|
65
|
+
if not 0 <= overlap < size:
|
|
66
|
+
raise ValueError("overlap must be >= 0 and smaller than size")
|
|
67
|
+
text = text.strip()
|
|
68
|
+
if len(text) <= size:
|
|
69
|
+
return [text] if text else []
|
|
70
|
+
pieces = _split(text, size, 0)
|
|
71
|
+
return _merge(pieces, size, overlap)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _split(text: str, size: int, sep_idx: int) -> list[str]:
|
|
75
|
+
if len(text) <= size:
|
|
76
|
+
return [text]
|
|
77
|
+
if sep_idx >= len(_SEPARATORS):
|
|
78
|
+
return [text[i : i + size] for i in range(0, len(text), size)]
|
|
79
|
+
sep = _SEPARATORS[sep_idx]
|
|
80
|
+
parts = text.split(sep)
|
|
81
|
+
if len(parts) == 1:
|
|
82
|
+
return _split(text, size, sep_idx + 1)
|
|
83
|
+
out: list[str] = []
|
|
84
|
+
for i, part in enumerate(parts):
|
|
85
|
+
piece = part + (sep if i < len(parts) - 1 else "")
|
|
86
|
+
if len(piece) > size:
|
|
87
|
+
out.extend(_split(piece, size, sep_idx + 1))
|
|
88
|
+
elif piece:
|
|
89
|
+
out.append(piece)
|
|
90
|
+
return out
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _merge(pieces: list[str], size: int, overlap: int) -> list[str]:
|
|
94
|
+
chunks: list[str] = []
|
|
95
|
+
current = ""
|
|
96
|
+
for piece in pieces:
|
|
97
|
+
if current and len(current) + len(piece) > size:
|
|
98
|
+
chunks.append(current.strip())
|
|
99
|
+
# Carry over up to `overlap` characters, shrunk if needed so tail + next piece still fits in `size`
|
|
100
|
+
# (dropping the overlap entirely would lose context at exactly the boundaries that need it).
|
|
101
|
+
room = min(overlap, size - len(piece))
|
|
102
|
+
tail = current[-room:] if room > 0 else ""
|
|
103
|
+
# Start the overlap at a word boundary so chunks don't begin mid-word.
|
|
104
|
+
if tail and " " in tail:
|
|
105
|
+
tail = tail[tail.index(" ") + 1 :]
|
|
106
|
+
current = tail
|
|
107
|
+
current += piece
|
|
108
|
+
if current.strip():
|
|
109
|
+
chunks.append(current.strip())
|
|
110
|
+
return [c for c in chunks if c]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def join_overlapping(parts: list[str], max_overlap: int = 400) -> str:
|
|
114
|
+
"""Re-join consecutive chunks of one section, dropping the text each chunk repeats from the previous one."""
|
|
115
|
+
out = ""
|
|
116
|
+
for part in parts:
|
|
117
|
+
if not out:
|
|
118
|
+
out = part
|
|
119
|
+
continue
|
|
120
|
+
best = 0
|
|
121
|
+
for n in range(min(len(out), len(part), max_overlap), 10, -1):
|
|
122
|
+
if out.endswith(part[:n]):
|
|
123
|
+
best = n
|
|
124
|
+
break
|
|
125
|
+
out += ("" if best else "\n") + part[best:]
|
|
126
|
+
return out
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def chunk_document(doc: Document, size: int = 800, overlap: int = 100) -> list[Chunk]:
|
|
130
|
+
"""Turn a document into chunks. Documents that already fit in one chunk stay whole."""
|
|
131
|
+
meta = doc.meta()
|
|
132
|
+
sections = split_sections(doc.text) if doc.extra.get("markdown", True) else [Section("", doc.text)]
|
|
133
|
+
if not sections:
|
|
134
|
+
return []
|
|
135
|
+
chunks: list[Chunk] = []
|
|
136
|
+
for section in sections:
|
|
137
|
+
for body in split_text(section.text, size, overlap):
|
|
138
|
+
n = len(chunks)
|
|
139
|
+
chunks.append(
|
|
140
|
+
Chunk(
|
|
141
|
+
id=f"{doc.id}#{n}",
|
|
142
|
+
doc_id=doc.id,
|
|
143
|
+
ordinal=n,
|
|
144
|
+
text=body,
|
|
145
|
+
heading=section.heading,
|
|
146
|
+
meta=meta,
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
return chunks
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Build the ``<context>`` block the model reads, and keep retrieved text in its place: data, never instructions.
|
|
2
|
+
|
|
3
|
+
Three defences, layered:
|
|
4
|
+
1. Every chunk is fenced in ``<chunk n=.. doc_id=..>`` tags inside ``<context>``, and any tag-like text inside a chunk
|
|
5
|
+
that could close the fence early is neutralised.
|
|
6
|
+
2. Chunks that look like prompt injection are flagged ``suspicious="true"`` and the system prompt tells the model to
|
|
7
|
+
treat them as content to report, not orders to follow.
|
|
8
|
+
3. The answer's citations are validated afterwards against the chunk numbers that were actually supplied.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import html
|
|
14
|
+
import re
|
|
15
|
+
from collections.abc import Sequence
|
|
16
|
+
|
|
17
|
+
from itops_rag.core.models import Hit
|
|
18
|
+
|
|
19
|
+
_FENCE_TAGS = re.compile(r"</?\s*(context|chunk|system|instructions?|question|user|assistant)\b[^>]*>", re.I)
|
|
20
|
+
|
|
21
|
+
_INJECTION_PATTERNS = [
|
|
22
|
+
r"ignore (all|any|the)? ?(previous|prior|above|earlier) (instructions|prompts?|rules)",
|
|
23
|
+
r"disregard (all|any|the)? ?(previous|prior|above) ",
|
|
24
|
+
r"\b(system|developer) (note|prompt|message|override)\s*:",
|
|
25
|
+
r"you are now\b",
|
|
26
|
+
r"new instructions?\s*:",
|
|
27
|
+
r"(send|email|share|reply with) (me |us )?(your|the user'?s?) (password|credentials|mfa|one[- ]time|otp|token)",
|
|
28
|
+
r"do not (tell|inform|mention).{0,40}(user|anyone)",
|
|
29
|
+
r"act as (an? )?(admin|administrator|root|system)",
|
|
30
|
+
]
|
|
31
|
+
_INJECTION = re.compile("|".join(f"(?:{p})" for p in _INJECTION_PATTERNS), re.I)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def looks_like_injection(text: str) -> bool:
|
|
35
|
+
return bool(_INJECTION.search(text))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def neutralise(text: str) -> str:
|
|
39
|
+
"""Make fence-like tags inert by escaping their angle brackets."""
|
|
40
|
+
return _FENCE_TAGS.sub(lambda m: html.escape(m.group(0)), text)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _attr(value: object) -> str:
|
|
44
|
+
return html.escape(str(value or ""), quote=True)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def build_context(hits: Sequence[Hit]) -> tuple[str, list[int]]:
|
|
48
|
+
"""Return the context block and the 1-based numbers of chunks flagged as suspicious."""
|
|
49
|
+
parts = ["<context>"]
|
|
50
|
+
suspicious: list[int] = []
|
|
51
|
+
for n, h in enumerate(hits, start=1):
|
|
52
|
+
m = h.chunk.meta
|
|
53
|
+
flag = looks_like_injection(h.chunk.text)
|
|
54
|
+
if flag:
|
|
55
|
+
suspicious.append(n)
|
|
56
|
+
attrs = (
|
|
57
|
+
f'n="{n}" doc_id="{_attr(h.doc_id)}" title="{_attr(m.get("title"))}" '
|
|
58
|
+
f'section="{_attr(h.chunk.heading)}" type="{_attr(m.get("doc_type"))}" '
|
|
59
|
+
f'system="{_attr(m.get("system"))}" owner="{_attr(m.get("owner"))}" updated="{_attr(m.get("updated"))}"'
|
|
60
|
+
)
|
|
61
|
+
if flag:
|
|
62
|
+
attrs += ' suspicious="true"'
|
|
63
|
+
parts.append(f"<chunk {attrs}>\n{neutralise(h.chunk.text)}\n</chunk>")
|
|
64
|
+
parts.append("</context>")
|
|
65
|
+
return "\n".join(parts), suspicious
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
_CITE = re.compile(r"\[(\d+(?:\s*[-–,;]\s*\d+)*)\]")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def extract_citations(text: str) -> list[int]:
|
|
72
|
+
"""All chunk numbers cited as ``[1]``, ``[2, 3]``, ``[2; 3]`` or ``[1-3]``, in first-seen order."""
|
|
73
|
+
seen: list[int] = []
|
|
74
|
+
for m in _CITE.finditer(text):
|
|
75
|
+
for part in re.split(r"[,;]", m.group(1)):
|
|
76
|
+
lo, _, hi = part.replace("–", "-").partition("-")
|
|
77
|
+
start, end = int(lo.strip()), int((hi or lo).strip())
|
|
78
|
+
for n in range(start, min(end, start + 50) + 1):
|
|
79
|
+
if n not in seen:
|
|
80
|
+
seen.append(n)
|
|
81
|
+
return seen
|
itops_rag/core/costs.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Price table, per-answer cost, and a JSONL ledger with a monthly budget cap.
|
|
2
|
+
|
|
3
|
+
Prices are USD per million tokens (first-party Claude API, November 2026). Cache reads bill at the listed read price
|
|
4
|
+
and cache writes (5-minute TTL) at 1.25x input. Update ``PRICES`` when list prices change. Previous-generation models
|
|
5
|
+
are listed because server-side fallbacks can serve an answer from them. An unknown model is charged at the most
|
|
6
|
+
expensive listed price (and logged), so the budget cap errs on the safe side.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
from dataclasses import asdict, dataclass
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from itops_rag.core.models import Usage
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True, slots=True)
|
|
23
|
+
class Price:
|
|
24
|
+
input: float
|
|
25
|
+
output: float
|
|
26
|
+
cache_read: float
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
PRICES: dict[str, Price] = {
|
|
30
|
+
"claude-opus-5-5": Price(4.00, 20.00, 0.20),
|
|
31
|
+
"claude-sonnet-5-5": Price(2.00, 10.00, 0.10),
|
|
32
|
+
"claude-haiku-5-5": Price(0.10, 0.50, 0.01),
|
|
33
|
+
"claude-fable-5-2": Price(8.00, 40.00, 0.20),
|
|
34
|
+
# possible fallback targets
|
|
35
|
+
"claude-opus-5": Price(5.00, 25.00, 0.50),
|
|
36
|
+
"claude-opus-4-8": Price(5.00, 25.00, 0.50),
|
|
37
|
+
"claude-sonnet-5": Price(2.00, 10.00, 0.20),
|
|
38
|
+
}
|
|
39
|
+
_MOST_EXPENSIVE = max(PRICES.values(), key=lambda p: p.output)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def cost_usd(
|
|
43
|
+
model: str, input_tokens: int, output_tokens: int, cache_read: int = 0, cache_write: int = 0
|
|
44
|
+
) -> float:
|
|
45
|
+
p = PRICES.get(model)
|
|
46
|
+
if p is None:
|
|
47
|
+
log.warning("no price for model %s; charging the highest listed price", model)
|
|
48
|
+
p = _MOST_EXPENSIVE
|
|
49
|
+
return (
|
|
50
|
+
input_tokens * p.input
|
|
51
|
+
+ output_tokens * p.output
|
|
52
|
+
+ cache_read * p.cache_read
|
|
53
|
+
+ cache_write * p.input * 1.25
|
|
54
|
+
) / 1_000_000
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class BudgetExceededError(RuntimeError):
|
|
58
|
+
pass
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class CostLedger:
|
|
62
|
+
"""Append-only JSONL of every model call. Small, greppable, and easy to ship to a log pipeline."""
|
|
63
|
+
|
|
64
|
+
def __init__(self, path: Path, monthly_budget_usd: float) -> None:
|
|
65
|
+
self.path = path
|
|
66
|
+
self.budget = monthly_budget_usd
|
|
67
|
+
|
|
68
|
+
def month_to_date(self, now: datetime | None = None) -> float:
|
|
69
|
+
now = now or datetime.now(UTC)
|
|
70
|
+
prefix = now.strftime("%Y-%m")
|
|
71
|
+
total = 0.0
|
|
72
|
+
if not self.path.exists():
|
|
73
|
+
return 0.0
|
|
74
|
+
for line in self.path.read_text(encoding="utf-8").splitlines():
|
|
75
|
+
try:
|
|
76
|
+
rec = json.loads(line)
|
|
77
|
+
except json.JSONDecodeError:
|
|
78
|
+
continue
|
|
79
|
+
if str(rec.get("ts", "")).startswith(prefix):
|
|
80
|
+
total += float(rec.get("cost_usd", 0.0))
|
|
81
|
+
return total
|
|
82
|
+
|
|
83
|
+
def check(self) -> None:
|
|
84
|
+
if self.budget > 0 and self.month_to_date() >= self.budget:
|
|
85
|
+
raise BudgetExceededError(
|
|
86
|
+
f"monthly budget of ${self.budget:.2f} reached; raise ITOPS_RAG_MONTHLY_BUDGET_USD or wait for next month"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def record(self, usage: Usage, purpose: str, prompt_id: str = "") -> None:
|
|
90
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
rec = {
|
|
92
|
+
"ts": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
93
|
+
"purpose": purpose,
|
|
94
|
+
"prompt_id": prompt_id,
|
|
95
|
+
**asdict(usage),
|
|
96
|
+
}
|
|
97
|
+
with self.path.open("a", encoding="utf-8") as f:
|
|
98
|
+
f.write(json.dumps(rec) + "\n")
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Local embeddings. No API key and no text leaves the machine.
|
|
2
|
+
|
|
3
|
+
``FastEmbedder`` runs sentence-transformers/all-MiniLM-L6-v2 (384 dimensions) through ONNX Runtime via fastembed,
|
|
4
|
+
which gives the same vectors as the HuggingFace model without installing torch. ``HashEmbedder`` is a deterministic,
|
|
5
|
+
model-free stand-in for tests and dry runs.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Iterable, Sequence
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Protocol
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import numpy.typing as npt
|
|
18
|
+
|
|
19
|
+
Vector = npt.NDArray[np.float32]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Embedder(Protocol):
|
|
23
|
+
name: str
|
|
24
|
+
dim: int
|
|
25
|
+
|
|
26
|
+
def embed_documents(self, texts: Sequence[str]) -> Vector: ...
|
|
27
|
+
|
|
28
|
+
def embed_query(self, text: str) -> Vector: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _normalise(m: npt.NDArray[Any]) -> Vector:
|
|
32
|
+
m = np.asarray(m, dtype=np.float32)
|
|
33
|
+
norms = np.linalg.norm(m, axis=-1, keepdims=True)
|
|
34
|
+
norms[norms == 0] = 1.0
|
|
35
|
+
out: Vector = (m / norms).astype(np.float32)
|
|
36
|
+
return out
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class FastEmbedder:
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
model: str = "sentence-transformers/all-MiniLM-L6-v2",
|
|
43
|
+
cache_dir: Path | None = None,
|
|
44
|
+
batch_size: int = 64,
|
|
45
|
+
) -> None:
|
|
46
|
+
self.name = model
|
|
47
|
+
self._cache_dir = cache_dir
|
|
48
|
+
self._batch_size = batch_size
|
|
49
|
+
self._model: Any = None
|
|
50
|
+
self.dim = self._lookup_dim(model)
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def _lookup_dim(model: str) -> int:
|
|
54
|
+
from fastembed import TextEmbedding
|
|
55
|
+
|
|
56
|
+
for m in TextEmbedding.list_supported_models():
|
|
57
|
+
if m["model"].lower() == model.lower():
|
|
58
|
+
return int(m["dim"])
|
|
59
|
+
raise ValueError(f"fastembed does not support embedding model {model!r}")
|
|
60
|
+
|
|
61
|
+
def _load(self) -> Any:
|
|
62
|
+
if self._model is None:
|
|
63
|
+
from fastembed import TextEmbedding
|
|
64
|
+
|
|
65
|
+
self._model = TextEmbedding(
|
|
66
|
+
self.name, cache_dir=str(self._cache_dir) if self._cache_dir else None
|
|
67
|
+
)
|
|
68
|
+
return self._model
|
|
69
|
+
|
|
70
|
+
def embed_documents(self, texts: Sequence[str]) -> Vector:
|
|
71
|
+
if not texts:
|
|
72
|
+
return np.zeros((0, self.dim), dtype=np.float32)
|
|
73
|
+
vecs = list(self._load().embed(list(texts), batch_size=self._batch_size))
|
|
74
|
+
return _normalise(np.vstack(vecs))
|
|
75
|
+
|
|
76
|
+
def embed_query(self, text: str) -> Vector:
|
|
77
|
+
return _normalise(np.asarray(next(iter(self._load().query_embed(text))), dtype=np.float32))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_TOKEN = re.compile(r"[a-z0-9]+")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class HashEmbedder:
|
|
84
|
+
"""Bag of hashed word and character-trigram features. Deterministic, instant, good enough for unit tests."""
|
|
85
|
+
|
|
86
|
+
def __init__(self, dim: int = 256) -> None:
|
|
87
|
+
self.name = f"hash-{dim}"
|
|
88
|
+
self.dim = dim
|
|
89
|
+
|
|
90
|
+
def _features(self, text: str) -> Iterable[str]:
|
|
91
|
+
words = _TOKEN.findall(text.lower())
|
|
92
|
+
yield from words
|
|
93
|
+
for w in words:
|
|
94
|
+
padded = f"#{w}#"
|
|
95
|
+
yield from (padded[i : i + 3] for i in range(len(padded) - 2))
|
|
96
|
+
|
|
97
|
+
def _vec(self, text: str) -> Vector:
|
|
98
|
+
v = np.zeros(self.dim, dtype=np.float32)
|
|
99
|
+
for f in self._features(text):
|
|
100
|
+
h = int.from_bytes(hashlib.blake2b(f.encode(), digest_size=8).digest(), "little")
|
|
101
|
+
v[h % self.dim] += 1.0 if (h >> 63) & 1 else -1.0
|
|
102
|
+
return v
|
|
103
|
+
|
|
104
|
+
def embed_documents(self, texts: Sequence[str]) -> Vector:
|
|
105
|
+
if not texts:
|
|
106
|
+
return np.zeros((0, self.dim), dtype=np.float32)
|
|
107
|
+
return _normalise(np.vstack([self._vec(t) for t in texts]))
|
|
108
|
+
|
|
109
|
+
def embed_query(self, text: str) -> Vector:
|
|
110
|
+
return _normalise(self._vec(text))
|