sp2react 0.1.0__py3-none-any.whl

sp2react/dbmap.py

@@ -0,0 +1,245 @@
+"""Map SharePoint lists onto an existing database (the data your team already migrated).
+
+  reflect   read the database SCHEMA only (tables, columns, types, keys); no rows are read
+  draft     deterministic first pass: names, types and foreign keys
+  agent     the dbmap agent resolves what the draft could not (renamed or split columns, odd names)
+  validate  every mapped table/column exists, types are compatible, required fields are mapped
+The result is db-mapping.json plus db-mapping.md for you to review before code generation uses it.
+"""
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any, Optional
+
+TYPE_FAMILIES = {
+    "text": ("char", "text", "string", "clob", "varchar", "nvarchar", "uniqueidentifier"),
+    "number": ("int", "dec", "num", "float", "double", "real", "money"),
+    "date": ("date", "time"),
+    "boolean": ("bool", "bit", "tinyint"),
+    "binary": ("blob", "binary", "image", "bytea"),
+}
+FIELD_FAMILY = {"text": "text", "note": "text", "choice": "text", "multichoice": "text", "url": "text", "calculated": "text",
+                "taxonomy": "text", "number": "number", "currency": "number", "date": "date", "datetime": "date",
+                "boolean": "boolean", "lookup": "number", "user": "any"}
+SYSTEM_PATTERNS = {
+    "Created": r"^(created|created_?(at|on|date|dt|time)|create_?date|date_?created)$",
+    "Modified": r"^(modified|modified_?(at|on|date|dt|time)|updated_?(at|on)|last_?modified|date_?modified)$",
+    "Author": r"^(author|created_?by|creator)(_?id|_?login)?$",
+    "Editor": r"^(editor|modified_?by|updated_?by|last_?modified_?by)(_?id|_?login)?$",
+    "version": r"^(version|row_?version|ui_?version|rowversion|etag)$",
+}
+
+
+def tokens(name: str) -> set[str]:
+    s = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", name)
+    s = re.sub(r"_x0020_", " ", s)
+    parts = [p for p in re.split(r"[^A-Za-z0-9]+", s.lower()) if p]
+    out = set()
+    for p in parts:
+        if p in ("tbl", "t", "sp", "list", "lst", "dbo"):
+            continue
+        out.add(p[:-1] if p.endswith("s") and len(p) > 3 else p)
+    return out
+
+
+def similarity(a: str, b: str) -> float:
+    ta, tb = tokens(a), tokens(b)
+    if not ta or not tb:
+        return 0.0
+    return len(ta & tb) / len(ta | tb)
+
+
+def family(sql_type: str) -> str:
+    t = sql_type.lower()
+    for fam, keys in TYPE_FAMILIES.items():
+        if any(k in t for k in keys):
+            return "boolean" if fam == "number" and "tinyint(1)" in t else fam
+    return "text"
+
+
+def compatible(field_type: str, sql_type: str) -> bool:
+    want = FIELD_FAMILY.get(field_type, "text")
+    have = family(sql_type)
+    if want == "any":
+        return have in ("number", "text")
+    if want == "boolean":
+        return have in ("boolean", "number")
+    if want == "date":
+        return have in ("date", "text")
+    return have == want or (want == "text" and have in ("text",)) or (want == "number" and have == "number")
+
+
+# ------------------------------------------------------------------ reflect
+def reflect(url: str, schema: Optional[str] = None) -> dict:
+    try:
+        from sqlalchemy import create_engine, inspect
+    except ImportError as e:  # pragma: no cover - optional extra
+        raise RuntimeError("map-db needs SQLAlchemy and your driver: pip install 'sp2react[db]' (and pymysql or pyodbc)") from e
+    eng = create_engine(url, future=True)
+    ins = inspect(eng)
+    out: dict = {"dialect": eng.dialect.name, "schema": schema, "tables": {}}
+    for t in ins.get_table_names(schema=schema):
+        pk = ins.get_pk_constraint(t, schema=schema).get("constrained_columns") or []
+        fks = {}
+        for fk in ins.get_foreign_keys(t, schema=schema):
+            for c, rc in zip(fk.get("constrained_columns", []), fk.get("referred_columns", [])):
+                fks[c] = {"table": fk.get("referred_table"), "column": rc}
+        cols = [{"name": c["name"], "type": str(c["type"]), "nullable": bool(c.get("nullable", True)),
+                 "pk": c["name"] in pk, **({"fk": fks[c["name"]]} if c["name"] in fks else {})}
+                for c in ins.get_columns(t, schema=schema)]
+        out["tables"][t] = {"columns": cols, "pk": pk}
+    eng.dispose()
+    return out
+
+
+# -------------------------------------------------------------------- draft
+def draft(lists: dict[str, dict], db: dict) -> tuple[dict, list[str]]:
+    """lists: {ListKey: {"title", "fields": [...]}} -> (mapping, unresolved notes)."""
+    mapping: dict = {"version": 1, "schema": db.get("schema"), "lists": {}}
+    notes: list[str] = []
+    tables = db["tables"]
+    chosen_tables: dict[str, str] = {}
+    for key, l in lists.items():
+        scored = sorted(((max(similarity(key, t), similarity(l["title"], t)), t) for t in tables), reverse=True)
+        if not scored or scored[0][0] < 0.5:
+            notes.append(f"{key}: no table with a matching name (best: {scored[0][1] if scored else 'none'})")
+            continue
+        chosen_tables[key] = scored[0][1]
+    for key, tname in chosen_tables.items():
+        l, t = lists[key], tables[tname]
+        cols = {c["name"]: c for c in t["columns"]}
+        used: set[str] = set()
+        m: dict = {"table": tname, "key": (t["pk"][0] if t["pk"] else next((c for c in cols if c.lower() in ("id", "item_id", f"{tname}_id")), None)), "fields": {}, "system": {}}
+        if not m["key"]:
+            notes.append(f"{key}: table {tname} has no primary key; set 'key' to the column holding the item id")
+        used.add(m["key"])
+        for f in l["fields"]:
+            cands = []
+            for cname, c in cols.items():
+                if cname in used:
+                    continue
+                s = max(similarity(f["name"], cname), similarity(f["label"], cname))
+                if f["type"] == "lookup":
+                    target = chosen_tables.get(f["lookup"]["list"])
+                    if c.get("fk", {}).get("table") == target and target:
+                        s += 1.0
+                    if cname.lower().endswith("_id") or cname.lower().endswith("id"):
+                        s += 0.2
+                if f["type"] == "user" and re.search(r"(login|user|email|_id)$", cname.lower()):
+                    s += 0.1
+                if s > 0 and compatible(f["type"], c["type"]):
+                    cands.append((s, cname))
+            cands.sort(reverse=True)
+            if cands and cands[0][0] >= 0.5:
+                cname = cands[0][1]
+                used.add(cname)
+                spec: Any = cname
+                if f["type"] == "user" and family(cols[cname]["type"]) == "text":
+                    spec = {"column": cname, "ref": "login"}
+                if f.get("multi") or f["type"] == "multichoice":
+                    spec = {"column": cname, "delimiter": ";", **({"ref": "login"} if isinstance(spec, dict) else {})}
+                m["fields"][f["name"]] = spec
+            else:
+                notes.append(f"{key}.{f['name']} ({f['label']}, {f['type']}): no confident column match in {tname}")
+        for sysname, pat in SYSTEM_PATTERNS.items():
+            hit = next((c for c in cols if c not in used and re.match(pat, c.lower())), None)
+            if hit:
+                m["system"][sysname] = hit
+                used.add(hit)
+        mapping["lists"][key] = m
+    users = next((t for t in tables if re.search(r"(^|_)(app_?)?users?$", t.lower())), None)
+    if users:
+        cols = [c["name"] for c in tables[users]["columns"]]
+        pick = lambda pat: next((c for c in cols if re.search(pat, c.lower())), None)
+        um = {"table": users, "id": tables[users]["pk"][0] if tables[users]["pk"] else pick(r"^(user_?)?id$"),
+              "login": pick(r"(login|user_?name|account|upn|sam)"), "title": pick(r"(display|full_?name|^name$|title)"), "email": pick(r"mail")}
+        if um["id"] and um["login"]:
+            mapping["users"] = {k: v for k, v in um.items() if v}
+    return mapping, notes
+
+
+# ----------------------------------------------------------------- validate
+def validate(mapping: dict, lists: dict[str, dict], db: dict) -> tuple[list[str], list[str]]:
+    """-> (problems that block, warnings)."""
+    problems, warnings = [], []
+    tables = db["tables"]
+    for key, l in lists.items():
+        m = mapping.get("lists", {}).get(key)
+        if not m:
+            problems.append(f"List {key} ({l['title']}) is not mapped to a table")
+            continue
+        t = tables.get(m.get("table"))
+        if not t:
+            problems.append(f"{key}: table '{m.get('table')}' does not exist")
+            continue
+        cols = {c["name"]: c for c in t["columns"]}
+        if m.get("key") not in cols:
+            problems.append(f"{key}: key column '{m.get('key')}' does not exist in {m['table']}")
+        for fname, spec in m.get("fields", {}).items():
+            cname = spec if isinstance(spec, str) else spec.get("column")
+            f = next((x for x in l["fields"] if x["name"] == fname), None)
+            if not f:
+                problems.append(f"{key}: '{fname}' is not a field of the list")
+                continue
+            if cname not in cols:
+                problems.append(f"{key}.{fname}: column '{cname}' does not exist in {m['table']}")
+                continue
+            if not compatible(f["type"], cols[cname]["type"]):
+                warnings.append(f"{key}.{fname}: {f['type']} field mapped to {cols[cname]['type']} column {cname}")
+            if f.get("required") and cols[cname]["nullable"] is False:
+                pass
+            if f["type"] == "lookup" and f["lookup"]["list"] not in mapping.get("lists", {}):
+                warnings.append(f"{key}.{fname}: looks up {f['lookup']['list']}, which is not mapped (titles will be empty)")
+        for f in l["fields"]:
+            if f["name"] not in m.get("fields", {}):
+                (problems if f.get("required") else warnings).append(
+                    f"{key}.{f['name']} ({f['label']}{', required' if f.get('required') else ''}) has no column")
+        for sysname, cname in m.get("system", {}).items():
+            if cname not in cols:
+                problems.append(f"{key}: system column {sysname} -> '{cname}' does not exist")
+        for cname, c in cols.items():
+            mapped = {m.get("key")} | {(s if isinstance(s, str) else s.get("column")) for s in m.get("fields", {}).values()} | set(m.get("system", {}).values())
+            if cname not in mapped and not c["nullable"] and not c["pk"]:
+                warnings.append(f"{key}: NOT NULL column {m['table']}.{cname} is not mapped; inserts will fail unless it has a default")
+    um = mapping.get("users")
+    if um:
+        t = tables.get(um.get("table"))
+        if not t:
+            problems.append(f"users: table '{um.get('table')}' does not exist")
+        else:
+            cols = {c["name"] for c in t["columns"]}
+            for k in ("id", "login"):
+                if um.get(k) not in cols:
+                    problems.append(f"users: {k} column '{um.get(k)}' does not exist")
+    return problems, warnings
+
+
+def report(mapping: dict, lists: dict[str, dict], db: dict, problems: list[str], warnings: list[str]) -> str:
+    lines = ["# Database mapping: SharePoint lists -> existing tables", "",
+             f"Database: {db.get('dialect')} {('schema ' + db['schema']) if db.get('schema') else ''}. Only the schema was read; no rows.", "",
+             f"**{'Ready' if not problems else 'Needs attention'}**: {len(problems)} problem(s), {len(warnings)} warning(s).", ""]
+    for key, l in lists.items():
+        m = mapping.get("lists", {}).get(key)
+        lines += [f"## {l['title']} ({key}) -> `{m['table'] if m else 'not mapped'}`", ""]
+        if not m:
+            continue
+        cols = {c["name"]: c for c in db["tables"].get(m["table"], {}).get("columns", [])}
+        lines += [f"Key: `{m.get('key')}`", "", "| Field | Type | Column | Column type |", "|---|---|---|---|"]
+        for f in l["fields"]:
+            spec = m.get("fields", {}).get(f["name"])
+            cname = spec if isinstance(spec, str) else (spec or {}).get("column")
+            extra = f" ({', '.join(f'{k}={v}' for k, v in spec.items() if k != 'column')})" if isinstance(spec, dict) else ""
+            lines.append(f"| {f['label']} (`{f['name']}`) | {f['type']} | {('`' + cname + '`' + extra) if cname else '**missing**'} | {cols.get(cname, {}).get('type', '')} |")
+        if m.get("system"):
+            lines += ["", "System columns: " + ", ".join(f"{k} -> `{v}`" for k, v in m["system"].items())]
+        lines.append("")
+    if mapping.get("users"):
+        u = mapping["users"]
+        lines += ["## People", "", f"Users table `{u['table']}` (id `{u['id']}`, login `{u['login']}`" + (f", name `{u['title']}`" if u.get("title") else "") + ")", ""]
+    if problems:
+        lines += ["## Problems (block code generation)", ""] + [f"- {p}" for p in problems] + [""]
+    if warnings:
+        lines += ["## Warnings", ""] + [f"- {w}" for w in warnings]
+    return "\n".join(lines) + "\n"

sp2react/llm/providers.py

@@ -0,0 +1,219 @@
+"""Model providers. Bridger (GitHub Copilot bridge) serving Claude is the default.
+
+Two wire formats cover every provider:
+  * Anthropic Messages  (POST {base}/v1/messages)          -> anthropic, bridger(api_format=anthropic)
+  * OpenAI Chat         (POST {base}/v1/chat/completions)  -> openai, bridger(api_format=openai), azure_openai
+"""
+from __future__ import annotations
+
+import logging
+import random
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Callable, Optional
+
+import requests
+
+from ..config import ConfigError, LLMConfig, secret
+
+log = logging.getLogger(__name__)
+RETRY_STATUS = {408, 409, 425, 429, 500, 502, 503, 504, 529}
+
+
+class LLMError(RuntimeError):
+    pass
+
+
+@dataclass
+class Completion:
+    text: str
+    model: str = ""
+    input_tokens: int = 0
+    output_tokens: int = 0
+    truncated: bool = False          # stopped because of max_tokens
+    raw: dict = field(default_factory=dict, repr=False)
+
+
+class LLM:
+    name = "base"
+
+    def __init__(self, cfg: LLMConfig):
+        self.cfg = cfg
+        self.policy = None
+        self._local = threading.local()
+
+    @property
+    def session(self) -> requests.Session:
+        s = getattr(self._local, "session", None)
+        if s is None:
+            s = requests.Session()
+            s.verify = self.cfg.verify_ssl
+            if self.policy is not None:
+                self.policy.guard(s)
+            self._local.session = s
+        return s
+
+    def _post(self, url: str, headers: dict, body: dict, timeout: Optional[int]) -> dict:
+        delay = 2.0
+        for attempt in range(5):
+            try:
+                r = self.session.post(url, headers=headers, json=body, timeout=timeout or self.cfg.timeout)
+            except requests.RequestException as e:
+                if attempt == 4:
+                    raise LLMError(f"{self.name}: request failed: {e}") from e
+                log.warning("%s: %s, retrying in %.0fs", self.name, e, delay)
+            else:
+                if r.status_code < 400:
+                    try:
+                        return r.json()
+                    except ValueError as e:
+                        raise LLMError(f"{self.name}: non-JSON response: {r.text[:300]}") from e
+                if r.status_code not in RETRY_STATUS or attempt == 4:
+                    raise LLMError(f"{self.name}: HTTP {r.status_code}: {r.text[:500]}")
+                ra = r.headers.get("retry-after")
+                if ra and ra.replace(".", "", 1).isdigit():
+                    delay = max(delay, float(ra))
+                log.warning("%s: HTTP %s, retrying in %.0fs", self.name, r.status_code, delay)
+            time.sleep(delay + random.random())
+            delay = min(delay * 2, 60)
+        raise LLMError(f"{self.name}: gave up")  # pragma: no cover
+
+    def complete(self, system: str, messages: list[dict], *, max_tokens: Optional[int] = None,
+                 json_output: bool = False, timeout: Optional[int] = None) -> Completion:
+        raise NotImplementedError
+
+    def models(self) -> list[str]:
+        return []
+
+
+def _key(cfg: LLMConfig, required: bool) -> Optional[str]:
+    return secret(cfg.api_key_env, "model API key/token", required=required) if cfg.api_key_env else None
+
+
+class AnthropicMessages(LLM):
+    """Anthropic Messages API: api.anthropic.com, or Bridger serving Claude in Anthropic format."""
+
+    def __init__(self, cfg: LLMConfig, base_url: str, name: str, bearer: bool):
+        super().__init__(cfg)
+        self.base = base_url.rstrip("/").removesuffix("/v1")
+        self.name = name
+        self.key = _key(cfg, required=not bearer)
+        self.bearer = bearer
+        if not cfg.model:
+            raise ConfigError(f"{name}: set llm.model (run `sp2react doctor` to list models)")
+
+    def _headers(self) -> dict:
+        h = {"content-type": "application/json", "anthropic-version": "2023-06-01", **self.cfg.extra_headers}
+        if self.key:
+            h["x-api-key"] = self.key
+            if self.bearer:
+                h["authorization"] = f"Bearer {self.key}"
+        return h
+
+    def complete(self, system, messages, *, max_tokens=None, json_output=False, timeout=None) -> Completion:
+        body = {"model": self.cfg.model, "max_tokens": max_tokens or self.cfg.max_tokens, "system": system,
+                "messages": [{"role": m["role"], "content": m["content"]} for m in messages],
+                "temperature": self.cfg.temperature}
+        data = self._post(f"{self.base}/v1/messages", self._headers(), body, timeout)
+        text = "".join(b.get("text", "") for b in data.get("content", []) if b.get("type") == "text")
+        u = data.get("usage") or {}
+        return Completion(text=text, model=data.get("model", self.cfg.model or ""), input_tokens=u.get("input_tokens", 0),
+                          output_tokens=u.get("output_tokens", 0), truncated=data.get("stop_reason") == "max_tokens", raw=data)
+
+    def models(self) -> list[str]:
+        r = self.session.get(f"{self.base}/v1/models", headers=self._headers(), timeout=30)
+        r.raise_for_status()
+        return [m.get("id") for m in r.json().get("data", [])]
+
+
+class OpenAIChat(LLM):
+    """OpenAI-style chat completions: OpenAI, Bridger in OpenAI format, Azure OpenAI."""
+
+    def __init__(self, cfg: LLMConfig, base_url: str, name: str, azure: bool = False, key_required: bool = True):
+        super().__init__(cfg)
+        self.name = name
+        self.azure = azure
+        self.key = _key(cfg, required=key_required)
+        base = base_url.rstrip("/")
+        if azure:
+            if not cfg.deployment:
+                raise ConfigError("azure_openai: set llm.deployment")
+            self.url = f"{base}/openai/deployments/{cfg.deployment}/chat/completions?api-version={cfg.api_version}"
+            self.models_url = None
+        else:
+            if not cfg.model:
+                raise ConfigError(f"{name}: set llm.model (run `sp2react doctor` to list models)")
+            root = base if base.endswith("/v1") else base + "/v1"
+            self.url, self.models_url = f"{root}/chat/completions", f"{root}/models"
+
+    def _headers(self) -> dict:
+        h = {"content-type": "application/json", **self.cfg.extra_headers}
+        if self.key:
+            h["api-key" if self.azure else "authorization"] = self.key if self.azure else f"Bearer {self.key}"
+        return h
+
+    def complete(self, system, messages, *, max_tokens=None, json_output=False, timeout=None) -> Completion:
+        body = {"messages": [{"role": "system", "content": system}] + messages,
+                "max_tokens": max_tokens or self.cfg.max_tokens, "temperature": self.cfg.temperature}
+        if not self.azure:
+            body["model"] = self.cfg.model
+        if json_output and self.cfg.json_mode:
+            body["response_format"] = {"type": "json_object"}
+        data = self._post(self.url, self._headers(), body, timeout)
+        choice = (data.get("choices") or [{}])[0]
+        u = data.get("usage") or {}
+        return Completion(text=(choice.get("message") or {}).get("content") or "", model=data.get("model", self.cfg.model or ""),
+                          input_tokens=u.get("prompt_tokens", 0), output_tokens=u.get("completion_tokens", 0),
+                          truncated=choice.get("finish_reason") == "length", raw=data)
+
+    def models(self) -> list[str]:
+        if not self.models_url:
+            return [self.cfg.deployment or ""]
+        r = self.session.get(self.models_url, headers=self._headers(), timeout=30)
+        r.raise_for_status()
+        return [m.get("id") for m in r.json().get("data", [])]
+
+
+class ScriptedLLM(LLM):
+    """Deterministic stand-in for tests and dry runs: responder(agent, system, messages) -> text."""
+    name = "scripted"
+
+    def __init__(self, responder: Callable[[str, str, list[dict]], str], cfg: Optional[LLMConfig] = None):
+        super().__init__(cfg or LLMConfig(provider="scripted"))
+        self.responder = responder
+        self.calls: list[dict] = []
+        self.agent = ""
+
+    def complete(self, system, messages, *, max_tokens=None, json_output=False, timeout=None) -> Completion:
+        agent = self.agent
+        for line in system.splitlines()[:3]:
+            if line.startswith("<agent name=\""):
+                agent = line.split('"')[1]
+        self.calls.append({"agent": agent, "messages": messages})
+        return Completion(text=self.responder(agent, system, messages), model="scripted")
+
+
+def make_llm(cfg: LLMConfig, policy=None) -> LLM:
+    llm = _make(cfg)
+    llm.policy = policy
+    return llm
+
+
+def _make(cfg: LLMConfig) -> LLM:
+    p = cfg.provider
+    if p == "bridger":
+        if not cfg.base_url:
+            raise ConfigError("bridger: set llm.base_url to the Bridger endpoint, e.g. http://localhost:4141")
+        if cfg.api_format == "anthropic":
+            return AnthropicMessages(cfg, cfg.base_url, "bridger", bearer=True)
+        return OpenAIChat(cfg, cfg.base_url, "bridger", key_required=False)
+    if p == "anthropic":
+        return AnthropicMessages(cfg, cfg.base_url or "https://api.anthropic.com", "anthropic", bearer=False)
+    if p == "openai":
+        return OpenAIChat(cfg, cfg.base_url or "https://api.openai.com/v1", "openai")
+    if p == "azure_openai":
+        if not cfg.base_url:
+            raise ConfigError("azure_openai: set llm.base_url to https://<resource>.openai.azure.com")
+        return OpenAIChat(cfg, cfg.base_url, "azure_openai", azure=True)
+    raise ConfigError("provider 'scripted' is only for tests; construct ScriptedLLM directly")