user-intent-discovery 1.1.0__py3-none-any.whl

user_intent_discovery/__init__.py

@@ -0,0 +1,46 @@
+"""user-intent-discovery: cluster chatbot questions and surface representative ones.
+
+Public API:
+    TopicClusterer   -- the orchestrator; call .fit(texts) to get results.
+    ClusteringResult -- the dataclass returned by fit().
+
+Split-compute across machines: pass any of the ``Remote*`` stages into
+``TopicClusterer`` (or use :func:`user_intent_discovery.remote.remote_stages` to build
+all four at once) to run individual stages on a different server::
+
+    from user_intent_discovery import TopicClusterer
+    from user_intent_discovery.remote import remote_stages
+
+    stages = remote_stages("http://gpu-box:8000", cluster_dim=5, viz_dim=3)
+    result = TopicClusterer(**stages, min_cluster_size=3).fit(questions)
+
+Everything else (custom embedders, summarizers, the API app factory) lives in
+submodules and can be imported directly, e.g.::
+
+    from user_intent_discovery.summarization import OllamaSummarizer
+    from user_intent_discovery.api import create_app
+"""
+
+from .client import TopicClustererClient
+from .pipeline import TopicClusterer
+from .remote import (
+    RemoteClusterer,
+    RemoteEmbedder,
+    RemoteReducer,
+    RemoteSummarizer,
+    remote_stages,
+)
+from .results import ClusteringResult
+
+__all__ = [
+    "TopicClusterer",
+    "TopicClustererClient",
+    "ClusteringResult",
+    "RemoteEmbedder",
+    "RemoteReducer",
+    "RemoteClusterer",
+    "RemoteSummarizer",
+    "remote_stages",
+]
+
+__version__ = "1.1.0"

user_intent_discovery/api/app.py

@@ -0,0 +1,53 @@
+"""FastAPI application factory."""
+
+from __future__ import annotations
+
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+
+from ..exceptions import OllamaConnectionError, ToolDiscoveryError
+from ..summarization import Summarizer
+from .routes import public_router, compute_router
+
+
+def create_app(
+    summarizer: Summarizer | None = None,
+    api_key: str | None = None,
+    cors_origins: list[str] | None = None,
+) -> FastAPI:
+    """Build the FastAPI app.
+
+    Args:
+        summarizer: If provided, the /summarize endpoint and UI tab are enabled.
+            If None, /summarize returns 503 and the UI greys the tab out.
+        api_key: If set, every compute endpoint requires ``X-API-Key`` to match.
+            ``/`` (UI) and ``/health`` remain public.
+        cors_origins: Allowed origins for cross-origin requests. ``None`` (the
+            default) allows any origin — appropriate for local dev; set an
+            explicit list in production.
+    """
+    app = FastAPI(title="user-intent-discovery", version="1.1.0")
+    app.state.summarizer = summarizer
+    app.state.api_key = api_key
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=cors_origins if cors_origins is not None else ["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    app.include_router(public_router)
+    app.include_router(compute_router)
+
+    @app.exception_handler(OllamaConnectionError)
+    def _ollama_down(request: Request, exc: OllamaConnectionError) -> JSONResponse:
+        return JSONResponse(status_code=503, content={"detail": str(exc)})
+
+    @app.exception_handler(ToolDiscoveryError)
+    def _tool_error(request: Request, exc: ToolDiscoveryError) -> JSONResponse:
+        return JSONResponse(status_code=500, content={"detail": str(exc)})
+
+    return app

user_intent_discovery/api/cli.py

@@ -0,0 +1,66 @@
+"""Console entry point: ``user-intent-discovery serve``."""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+import uvicorn
+
+from ..summarization import OllamaSummarizer
+from .app import create_app
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(prog="user-intent-discovery")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve = sub.add_parser("serve", help="Run the API + UI server.")
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8000)
+    serve.add_argument("--ollama-url", default="http://localhost:11434")
+    serve.add_argument("--ollama-model", default="llama3.2")
+    serve.add_argument(
+        "--no-summarizer",
+        action="store_true",
+        help="Disable summarization (UI summarize tab greyed out).",
+    )
+    serve.add_argument(
+        "--api-key",
+        default=None,
+        help=(
+            "Require this key in the X-API-Key header on compute endpoints. "
+            "Falls back to $USER_INTENT_DISCOVERY_API_KEY if unset. "
+            "/health and / remain public."
+        ),
+    )
+    serve.add_argument(
+        "--cors-origin",
+        action="append",
+        default=None,
+        dest="cors_origins",
+        help=(
+            "Allowed CORS origin (repeatable). Omit to allow any origin "
+            "(fine for local dev; set explicit origins in production)."
+        ),
+    )
+
+    args = parser.parse_args()
+
+    api_key = args.api_key or os.environ.get("USER_INTENT_DISCOVERY_API_KEY")
+
+    summarizer = (
+        None
+        if args.no_summarizer
+        else OllamaSummarizer(model=args.ollama_model, base_url=args.ollama_url)
+    )
+    app = create_app(
+        summarizer=summarizer,
+        api_key=api_key,
+        cors_origins=args.cors_origins,
+    )
+    uvicorn.run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

user_intent_discovery/api/routes.py

@@ -0,0 +1,279 @@
+"""HTTP endpoints. Each stage is exposed separately, plus a one-shot /pipeline.
+
+Split into two routers: ``public_router`` (UI + /health, always open) and
+``compute_router`` (every stage endpoint, guarded by an optional X-API-Key
+check driven by ``app.state.api_key``).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import asdict
+from pathlib import Path
+from urllib.parse import urlparse
+
+import numpy as np
+import requests
+from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from fastapi.responses import FileResponse
+
+from ..clustering import HdbscanClusterer
+from ..embedding import SentenceTransformerEmbedder
+from ..metrics import compute_metrics
+from ..pipeline import TopicClusterer
+from ..reduction import UmapReducer
+from ..representatives import reassign_noise as reassign_noise_fn
+from ..summarization import OllamaSummarizer, Summarizer
+from .schemas import (
+    ClusterRequest,
+    ClusterResponse,
+    EmbedRequest,
+    EmbedResponse,
+    HealthResponse,
+    IngestRequest,
+    IngestResponse,
+    MAX_INGEST_BODY_BYTES,
+    MAX_TEXTS,
+    PipelineRequest,
+    ReduceRequest,
+    ReduceResponse,
+    SummarizeRequest,
+    SummarizeResponse,
+)
+
+_STATIC = Path(__file__).parent / "static"
+# Fixed, server-controlled cache location — clients may only opt in/out,
+# never choose the path. Anchored to the project root, not the request.
+_CACHE_DIR = Path(__file__).resolve().parents[3] / "embedding_cache"
+
+
+def _get_summarizer(request: Request) -> Summarizer | None:
+    """Pull the summarizer the app was created with off app.state."""
+    return getattr(request.app.state, "summarizer", None)
+
+
+def _require_api_key(
+    request: Request,
+    x_api_key: str | None = Header(default=None, alias="X-API-Key"),
+) -> None:
+    """Reject the request when an api_key is configured and the header is wrong.
+
+    Applied to every compute route; no-op when no api_key is configured, so the
+    default open server keeps working without changes.
+    """
+    expected = getattr(request.app.state, "api_key", None)
+    if expected and x_api_key != expected:
+        raise HTTPException(status_code=401, detail="Invalid or missing X-API-Key.")
+
+
+public_router = APIRouter()
+compute_router = APIRouter(dependencies=[Depends(_require_api_key)])
+
+
+@public_router.get("/health", response_model=HealthResponse)
+def health(request: Request) -> HealthResponse:
+    return HealthResponse(
+        summarizer_available=_get_summarizer(request) is not None,
+        auth_required=getattr(request.app.state, "api_key", None) is not None,
+    )
+
+
+@public_router.get("/")
+def index() -> FileResponse:
+    """Serve the browser UI."""
+    return FileResponse(_STATIC / "index.html")
+
+
+@compute_router.post("/embed", response_model=EmbedResponse)
+def embed(req: EmbedRequest) -> EmbedResponse:
+    embedder = SentenceTransformerEmbedder(
+        model_name=req.model or "sentence-transformers/all-MiniLM-L6-v2",
+        normalize=req.normalize,
+    )
+    vectors = embedder.embed(req.texts)
+    return EmbedResponse(vectors=vectors.tolist(), dimension=embedder.dimension)
+
+
+@compute_router.post("/reduce", response_model=ReduceResponse)
+def reduce(req: ReduceRequest) -> ReduceResponse:
+    coords = UmapReducer(
+        n_components=req.target_dim,
+        n_neighbors=req.n_neighbors,
+        min_dist=req.min_dist,
+        metric=req.metric,
+        random_state=req.random_state,
+    ).fit_transform(np.asarray(req.vectors, dtype=np.float32))
+    return ReduceResponse(coords=coords.tolist())
+
+
+@compute_router.post("/cluster", response_model=ClusterResponse)
+def cluster(req: ClusterRequest) -> ClusterResponse:
+    coords = np.asarray(req.coords, dtype=np.float32)
+    labels, persistence = HdbscanClusterer(
+        min_cluster_size=req.min_cluster_size,
+        min_samples=req.min_samples,
+        cluster_selection_method=req.cluster_selection_method,
+        cluster_selection_epsilon=req.cluster_selection_epsilon,
+    ).fit(coords)
+
+    # Metrics computed on ORIGINAL labels (noise counted honestly).
+    metrics = compute_metrics(coords, labels, persistence)
+
+    out_labels = labels
+    if req.reassign_noise:
+        out_labels = reassign_noise_fn(coords, labels)
+
+    return ClusterResponse(
+        labels=[int(x) for x in out_labels],
+        persistence=persistence,
+        metrics=asdict(metrics),
+    )
+
+
+@compute_router.post("/pipeline")
+def pipeline(req: PipelineRequest) -> dict:
+    """One-shot: what the browser UI calls on submit. Never summarizes here."""
+    cache_dir = str(_CACHE_DIR) if req.cache_embeddings else None
+    clusterer = TopicClusterer(
+        embedding_model=req.embedding_model,
+        normalize_embeddings=req.normalize_embeddings,
+        embedding_cache_dir=cache_dir,
+        cluster_dim=req.cluster_dim,
+        umap_n_neighbors=req.umap_n_neighbors,
+        umap_min_dist=req.umap_min_dist,
+        umap_metric=req.umap_metric,
+        viz_dim=req.viz_dim,
+        min_cluster_size=req.min_cluster_size,
+        min_samples=req.min_samples,
+        cluster_selection_method=req.cluster_selection_method,
+        cluster_selection_epsilon=req.cluster_selection_epsilon,
+        top_n_per_cluster=req.top_n_per_cluster,
+        top_k_clusters=req.top_k_clusters,
+        representative_space=req.representative_space,
+        reassign_noise=req.reassign_noise,
+        summarizer=None,
+        random_state=req.random_state,
+    )
+    return clusterer.fit(req.texts).to_dict()
+
+
+@compute_router.post("/summarize", response_model=SummarizeResponse)
+def summarize(req: SummarizeRequest, request: Request) -> SummarizeResponse:
+    default = _get_summarizer(request)
+    if default is None and req.model is None:
+        raise HTTPException(status_code=503, detail="No summarizer configured.")
+
+    # Per-request override: build a summarizer for the named model, reusing the
+    # server's Ollama URL if the default is an OllamaSummarizer.
+    if req.model:
+        base_url = getattr(default, "base_url", "http://localhost:11434")
+        summarizer = OllamaSummarizer(model=req.model, base_url=base_url)
+    else:
+        summarizer = default
+
+    summaries = {
+        c.cluster_id: summarizer.summarize(c.cluster_id, c.representatives)
+        for c in req.clusters
+    }
+    return SummarizeResponse(summaries=summaries)
+
+
+# ---------- /ingest ----------
+# UI + backend-to-backend helper: fetch a URL server-side and hand back a
+# flat list of strings. Server-side because browsers block cross-origin
+# fetches (CORS) and cannot inject Authorization headers from the frontend.
+
+_TEXT_KEYS_IN_OBJECT = ("text", "question", "prompt", "query", "content")
+_LIST_KEYS_IN_ROOT = ("questions", "texts", "prompts", "queries", "items", "data")
+
+
+def _extract_texts_from_json(data: object) -> list[str]:
+    """Turn a decoded JSON body into a flat list of strings.
+
+    Accepts three shapes:
+      - ``["q1", "q2", ...]``
+      - ``[{"text": "q1"}, {"question": "q2"}, ...]`` (first hit wins)
+      - ``{"questions": [...]}`` / ``{"texts": [...]}`` / ``{"items": [...]}`` / ``{"data": [...]}``
+    """
+    if isinstance(data, list):
+        if all(isinstance(x, str) for x in data):
+            return [x.strip() for x in data if x and x.strip()]
+        out: list[str] = []
+        for item in data:
+            if isinstance(item, dict):
+                for k in _TEXT_KEYS_IN_OBJECT:
+                    v = item.get(k)
+                    if isinstance(v, str) and v.strip():
+                        out.append(v.strip())
+                        break
+        return out
+    if isinstance(data, dict):
+        for k in _LIST_KEYS_IN_ROOT:
+            if k in data:
+                return _extract_texts_from_json(data[k])
+    return []
+
+
+def _extract_texts_from_body(resp: requests.Response) -> list[str]:
+    """Try JSON first, fall back to comma/newline-splitting on plain text."""
+    try:
+        return _extract_texts_from_json(resp.json())
+    except ValueError:
+        pass
+    # Plain text: split on newlines and commas, strip and drop empties.
+    parts = re.split(r"[\n,]", resp.text)
+    return [p.strip() for p in parts if p.strip()]
+
+
+@compute_router.post("/ingest", response_model=IngestResponse)
+def ingest(req: IngestRequest) -> IngestResponse:
+    """Fetch texts from an external URL server-side (bypasses browser CORS)."""
+    parsed = urlparse(req.url)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(
+            status_code=400, detail="Only http:// and https:// URLs are allowed."
+        )
+
+    try:
+        # stream=True so we can enforce MAX_INGEST_BODY_BYTES before loading it all
+        with requests.get(
+            req.url,
+            headers=req.headers or {},
+            timeout=req.timeout,
+            stream=True,
+            allow_redirects=True,
+        ) as resp:
+            resp.raise_for_status()
+            # Reject bodies larger than the cap (checked incrementally so a
+            # malicious server can't stream forever).
+            content = bytearray()
+            for chunk in resp.iter_content(chunk_size=64 * 1024):
+                if not chunk:
+                    continue
+                content.extend(chunk)
+                if len(content) > MAX_INGEST_BODY_BYTES:
+                    raise HTTPException(
+                        status_code=413,
+                        detail=(
+                            f"Upstream response exceeded "
+                            f"{MAX_INGEST_BODY_BYTES // 1_000_000} MB cap."
+                        ),
+                    )
+            resp._content = bytes(content)  # so .json() / .text see the buffered body
+    except requests.exceptions.RequestException as exc:
+        raise HTTPException(status_code=502, detail=f"Upstream fetch failed: {exc}")
+
+    texts = _extract_texts_from_body(resp)
+    if not texts:
+        raise HTTPException(
+            status_code=422,
+            detail=(
+                "Could not extract any texts from the response. Expected a JSON list "
+                "of strings, a list of objects with a text/question/prompt field, an "
+                "object containing a 'questions'/'texts'/'items' list, or a "
+                "newline/comma-separated plain-text body."
+            ),
+        )
+    if len(texts) > MAX_TEXTS:
+        texts = texts[:MAX_TEXTS]   # cap silently; the UI will show the count.
+    return IngestResponse(texts=texts, count=len(texts))

user_intent_discovery/api/schemas.py

@@ -0,0 +1,138 @@
+"""Pydantic request/response models.
+
+Input caps are applied at the schema layer so garbage requests are rejected
+at the boundary (413/422) rather than exploding memory later. Bump the
+constants below if a real workload legitimately needs more headroom.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from pydantic import BaseModel, Field, StringConstraints
+
+# ---------- caps ----------
+MAX_TEXTS = 20_000              # per-request text count
+MAX_TEXT_LENGTH = 5_000         # per-text character count
+MAX_VECTORS = 50_000            # per-request vector count for /reduce, /cluster
+MAX_VECTOR_DIM = 4_096          # sanity cap on embedding dimensionality
+MAX_CLUSTERS_PER_SUMMARIZE = 500
+MAX_REPS_PER_CLUSTER = 100
+MAX_URL_LENGTH = 2_048          # for /ingest — usual browser limit
+MAX_INGEST_BODY_BYTES = 10 * 1024 * 1024   # 10 MB cap on the fetched body
+
+# A single non-empty text, whitespace-stripped, length-bounded.
+Text = Annotated[
+    str,
+    StringConstraints(
+        min_length=1, max_length=MAX_TEXT_LENGTH, strip_whitespace=True
+    ),
+]
+
+
+# ---------- /embed ----------
+class EmbedRequest(BaseModel):
+    texts: list[Text] = Field(min_length=1, max_length=MAX_TEXTS)
+    model: str | None = None
+    normalize: bool = True
+
+
+class EmbedResponse(BaseModel):
+    vectors: list[list[float]]
+    dimension: int
+
+
+# ---------- /reduce ----------
+class ReduceRequest(BaseModel):
+    vectors: list[list[float]] = Field(min_length=1, max_length=MAX_VECTORS)
+    target_dim: int = Field(default=5, ge=1, le=100)
+    n_neighbors: int = Field(default=15, ge=2, le=1_000)
+    min_dist: float = Field(default=0.0, ge=0.0, le=1.0)
+    metric: str = "cosine"
+    random_state: int | None = 42
+
+
+class ReduceResponse(BaseModel):
+    coords: list[list[float]]
+
+
+# ---------- /cluster ----------
+class ClusterRequest(BaseModel):
+    coords: list[list[float]] = Field(min_length=2, max_length=MAX_VECTORS)
+    min_cluster_size: int = Field(default=5, ge=2)
+    min_samples: int | None = None
+    cluster_selection_method: str = "eom"
+    cluster_selection_epsilon: float = Field(default=0.0, ge=0.0)
+    reassign_noise: bool = False
+
+
+class ClusterResponse(BaseModel):
+    labels: list[int]
+    persistence: dict[int, float]
+    metrics: dict
+
+
+# ---------- /pipeline ----------
+class PipelineRequest(BaseModel):
+    texts: list[Text] = Field(min_length=1, max_length=MAX_TEXTS)
+    # Embedding
+    embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2"
+    normalize_embeddings: bool = True
+    cache_embeddings: bool = False
+    # Reduction
+    cluster_dim: int = Field(default=5, ge=1, le=100)
+    umap_n_neighbors: int = Field(default=15, ge=2, le=1_000)
+    umap_min_dist: float = Field(default=0.0, ge=0.0, le=1.0)
+    umap_metric: str = "cosine"
+    viz_dim: int = Field(default=3, ge=0, le=100)
+    # Clustering
+    min_cluster_size: int = Field(default=5, ge=2)
+    min_samples: int | None = None
+    cluster_selection_method: str = "eom"
+    cluster_selection_epsilon: float = Field(default=0.0, ge=0.0)
+    # Representatives
+    top_n_per_cluster: int = Field(default=5, ge=1, le=1_000)
+    top_k_clusters: int | None = None
+    representative_space: str = "embedding"
+    # Noise
+    reassign_noise: bool = False
+    # Reproducibility
+    random_state: int | None = 42
+
+
+# ---------- /summarize ----------
+class ClusterToSummarize(BaseModel):
+    cluster_id: int
+    representatives: list[Text] = Field(
+        min_length=1, max_length=MAX_REPS_PER_CLUSTER
+    )
+
+
+class SummarizeRequest(BaseModel):
+    clusters: list[ClusterToSummarize] = Field(
+        min_length=1, max_length=MAX_CLUSTERS_PER_SUMMARIZE
+    )
+    model: str | None = None   # None = use the server's default summarizer
+
+
+class SummarizeResponse(BaseModel):
+    summaries: dict[int, str]
+
+
+# ---------- /ingest ----------
+class IngestRequest(BaseModel):
+    url: str = Field(min_length=1, max_length=MAX_URL_LENGTH)
+    headers: dict[str, str] | None = None   # e.g. {"Authorization": "Bearer …"}
+    timeout: float = Field(default=30.0, ge=1.0, le=120.0)
+
+
+class IngestResponse(BaseModel):
+    texts: list[str]
+    count: int
+
+
+# ---------- /health ----------
+class HealthResponse(BaseModel):
+    status: str = "ok"
+    summarizer_available: bool
+    auth_required: bool = False