sp2react 0.1.0__py3-none-any.whl

sp2react/__init__.py

@@ -0,0 +1,18 @@
+"""sp2react: convert classic SharePoint ASPX/jQuery list forms and pages into React + FastAPI with AI agents.
+
+Python API:
+    from sp2react import Pipeline, load_config
+    p = Pipeline(load_config("sp2react.yaml"))
+    p.run(pages=["SitePages/Dashboard.aspx"])
+"""
+__version__ = "0.1.0"
+
+
+def __getattr__(name):
+    if name == "Pipeline":
+        from .pipeline import Pipeline
+        return Pipeline
+    if name == "load_config":
+        from .config import load_config
+        return load_config
+    raise AttributeError(name)

sp2react/agents/core.py

@@ -0,0 +1,146 @@
+"""Agent runtime: prompt -> model -> parse -> validate -> retry with the problems, with transcripts on disk.
+
+Agents never write straight into the app: their output is parsed and validated (schema, coverage gate,
+import whitelist, compile/type checks) and only accepted when it passes.
+"""
+from __future__ import annotations
+
+import json
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+from ..llm.providers import LLM, LLMError
+
+PROMPTS = Path(__file__).parent / "prompts"
+
+
+class ParseError(ValueError):
+    pass
+
+
+class AgentFailed(RuntimeError):
+    def __init__(self, agent: str, task: str, problems: list[str], transcript: Path):
+        super().__init__(f"{agent}/{task} failed validation after retries: {problems[:3]}")
+        self.agent, self.task, self.problems, self.transcript = agent, task, problems, transcript
+
+
+@dataclass
+class AgentResult:
+    output: Any
+    attempts: int
+    input_tokens: int = 0
+    output_tokens: int = 0
+    seconds: float = 0.0
+    notes: Optional[str] = None
+    transcript: Optional[Path] = None
+
+
+def render_sections(sections: dict[str, str]) -> str:
+    parts = []
+    for tag, content in sections.items():
+        if content is None or content == "":
+            continue
+        parts.append(f"<{tag}>\n{content.strip()}\n</{tag}>")
+    return "\n\n".join(parts)
+
+
+def parse_json(text: str) -> dict:
+    t = text.strip()
+    t = re.sub(r"^```(?:json)?\s*|\s*```$", "", t)
+    start = t.find("{")
+    if start < 0:
+        raise ParseError("Your reply did not contain a JSON object. Reply with exactly one JSON object.")
+    try:
+        obj, _ = json.JSONDecoder().raw_decode(t[start:])
+    except json.JSONDecodeError as e:
+        raise ParseError(f"Your reply was not valid JSON ({e.msg} at line {e.lineno} column {e.colno}). Reply with exactly one valid JSON object.") from e
+    if not isinstance(obj, dict):
+        raise ParseError("The JSON must be an object.")
+    return obj
+
+
+FILE_BLOCK = re.compile(r'<file\s+path="([^"]+)"\s*>\n?(.*?)\n?</file>', re.S)
+
+
+def parse_files(text: str) -> dict:
+    files = {}
+    for path, body in FILE_BLOCK.findall(text):
+        body = re.sub(r"^\s*```\w*\n", "", body)
+        body = re.sub(r"\n```\s*$", "", body)
+        files[path.strip()] = body.rstrip() + "\n"
+    notes = re.search(r"<notes>(.*?)</notes>", text, re.S)
+    patch = re.search(r"<spec_patch>(.*?)</spec_patch>", text, re.S)
+    out = {"files": files, "notes": notes.group(1).strip() if notes else None, "spec_patch": None}
+    if patch:
+        out["spec_patch"] = parse_json(patch.group(1))
+    if not files and not patch:
+        raise ParseError('No <file path="...">...</file> blocks found. Return every file in that format.')
+    return out
+
+
+class Agent:
+    def __init__(self, name: str, llm: LLM, workdir: Path, output: str = "json", retries: int = 2,
+                 max_tokens: Optional[int] = None, progress: Callable[[str], None] = lambda s: None):
+        self.name, self.llm, self.output = name, llm, output
+        self.retries, self.max_tokens, self.progress = retries, max_tokens, progress
+        self.workdir = workdir / "agents" / name
+        self.prompt = (PROMPTS / f"{name}.md").read_text(encoding="utf-8")
+
+    @property
+    def system(self) -> str:
+        return f'<agent name="{self.name}">\n{self.prompt}\n</agent>'
+
+    def _complete(self, messages: list[dict]) -> tuple[str, int, int]:
+        text, tin, tout = "", 0, 0
+        msgs = list(messages)
+        for _ in range(4):
+            c = self.llm.complete(self.system, msgs, max_tokens=self.max_tokens, json_output=(self.output == "json"))
+            text += c.text
+            tin += c.input_tokens
+            tout += c.output_tokens
+            if not c.truncated:
+                break
+            msgs = messages + [{"role": "assistant", "content": text},
+                               {"role": "user", "content": "Your reply was cut off by the length limit. Continue exactly where it stopped. Do not repeat anything."}]
+        return text, tin, tout
+
+    def run(self, task_id: str, sections: dict[str, str], validate: Callable[[Any], list[str]]) -> AgentResult:
+        tdir = self.workdir / re.sub(r"[^\w.\-]+", "_", task_id)
+        tdir.mkdir(parents=True, exist_ok=True)
+        user = render_sections(sections)
+        (tdir / "prompt.md").write_text(f"# system\n\n{self.system}\n\n# user\n\n{user}\n", encoding="utf-8")
+        messages = [{"role": "user", "content": user}]
+        t0, tin, tout, problems = time.time(), 0, 0, []
+        for attempt in range(1, self.retries + 2):
+            self.progress(f"{self.name}: {task_id} attempt {attempt}")
+            try:
+                text, a, b = self._complete(messages)
+            except LLMError as e:
+                problems = [f"model call failed: {e}"]
+                (tdir / f"attempt-{attempt}.error.txt").write_text(str(e), encoding="utf-8")
+                if attempt > self.retries:
+                    break
+                continue
+            tin, tout = tin + a, tout + b
+            (tdir / f"attempt-{attempt}.response.md").write_text(text, encoding="utf-8")
+            try:
+                parsed = parse_json(text) if self.output == "json" else parse_files(text)
+                problems = validate(parsed)
+            except ParseError as e:
+                problems = [str(e)]
+            except Exception as e:  # a validator crashed on odd output: report it to the model like any problem
+                problems = [f"Output could not be processed: {type(e).__name__}: {e}"]
+            if not problems:
+                (tdir / "accepted.txt").write_text(f"attempt {attempt}\n", encoding="utf-8")
+                notes = parsed.get("notes") if isinstance(parsed, dict) else None
+                return AgentResult(parsed, attempt, tin, tout, time.time() - t0, notes, tdir)
+            (tdir / f"attempt-{attempt}.problems.md").write_text("\n".join(f"- {p}" for p in problems), encoding="utf-8")
+            messages = messages + [
+                {"role": "assistant", "content": text},
+                {"role": "user", "content": "<validation_feedback>\nYour output was rejected by automatic validation. Fix every problem and return the complete output again.\n"
+                                            + "\n".join(f"- {p}" for p in problems[:60]) + "\n</validation_feedback>"},
+            ]
+        raise AgentFailed(self.name, task_id, problems, tdir)

sp2react/agents/prompts/dbmap.md

@@ -0,0 +1,28 @@
+You are the database mapping agent in sp2react. A team has already migrated SharePoint list data into a relational database. The converted app must read and write those existing tables, so every SharePoint list and field needs to be mapped to the table and column that now holds it.
+
+You receive the SharePoint list schemas, the database schema (tables, columns, types, keys, foreign keys; no rows), a deterministic draft mapping and the problems it could not resolve.
+
+<rules>
+- Only use tables and columns that exist in <database_schema>. Never invent names.
+- A lookup field maps to the column holding the key of the target list's row (usually a foreign key). A person field maps to a column holding the users table id (default) or a login/email (then set "ref": "login").
+- Multi-value fields stored in one text column need "delimiter" (the separator used in the data, ";" if unknown).
+- "key" is the column that identifies an item (normally the primary key; it often still holds the SharePoint ID).
+- "system" maps Created, Modified, Author, Editor and version when such columns exist.
+- If a field truly has no column (it was dropped in the migration), leave it out and say so in "notes". Do not map a field to an unrelated column to make validation pass.
+- Keep every correct part of the draft unchanged.
+</rules>
+
+<mapping_format>
+{"version": 1, "schema": null,
+ "lists": {"<ListKey>": {"table": "...", "key": "...",
+                         "fields": {"<FieldInternalName>": "column" | {"column": "...", "ref": "login", "delimiter": ";"}},
+                         "system": {"Created": "...", "Modified": "...", "Author": "...", "Editor": "...", "version": "..."},
+                         "attachments": {"table": "...", "item": "...", "name": "...", "content": "..."}}},
+ "users": {"table": "...", "id": "...", "login": "...", "title": "...", "email": "...",
+           "groups": {"table": "...", "user": "...", "group": "..."}}}
+</mapping_format>
+
+<output_contract>
+Respond with one JSON object and nothing else: {"mapping": {...complete mapping...}, "notes": ["..."]}.
+If you receive <validation_feedback>, fix every problem and return the complete object again.
+</output_contract>

sp2react/agents/prompts/fix.md

@@ -0,0 +1,11 @@
+You are the fix agent in sp2react. The converted app failed a check: a TypeScript/Vite build, the generated API tests, the rule vector tests, or blocking review findings. You receive the failure output, the files involved and the relevant spec. Make the smallest change that fixes the failure while keeping strict parity with the legacy page.
+
+<constraints>
+- Only edit the files you are given under <files>. Scaffold files (components/sp, runtime, api.ts, backend/app/*.py except backend/app/custom/) are fixed infrastructure: if the failure is caused by them, say so in <notes> instead of editing them.
+- If the right fix is a spec change (a wrong rule), return the corrected spec patch as <spec_patch>{ "rules": [...replacement rules with the same ids...] }</spec_patch>.
+- Keep legacy texts verbatim. Do not delete behaviour to make a check pass.
+</constraints>
+
+<output_contract>
+Return complete replacement files as <file path="...">content</file>, optionally <spec_patch>...</spec_patch> and <notes>...</notes>. Nothing else.
+</output_contract>

sp2react/agents/prompts/handler.md

@@ -0,0 +1,22 @@
+You are the custom handler agent in sp2react. A spec rule of kind "custom" describes legacy SharePoint behaviour that the generic rule engine cannot express. Write the handler in both places so the browser and the API agree.
+
+<contracts>
+Frontend: frontend/src/custom/<handler>.ts
+  import type { Ctx, RuleError } from "../runtime/rules";
+  import type { Api } from "../api";
+  import type { Values } from "../runtime/spec";
+  export function onLoad(ctx: Ctx, api: Api): void                       // optional
+  export function onChange(field: string, ctx: Ctx): Values | void       // optional: return a patch of field values
+  export function validate(ctx: Ctx): RuleError[]                        // optional: {field, message, presentation: "inline"|"alert", rule}
+  ctx = {mode: "new"|"edit"|"display", values, original, user: {id, login, title, groups}}
+
+Backend: backend/app/custom/<handler>.py
+  def validate(ctx: dict) -> list[dict]        # optional, same error shape
+  def on_save(ctx: dict, item: dict, repo) -> None   # optional; repo.get(list, id), repo.all(list), repo.create(list, values, user), repo.update(list, id, values, user)
+  router = APIRouter()                         # optional, mounted under /api/custom
+Implement only the functions the rule needs. Keep legacy messages verbatim. Standard library + fastapi only on the backend; no new npm packages on the frontend.
+</contracts>
+
+<output_contract>
+Return each file as <file path="...">complete content</file>, nothing else. If you receive <validation_feedback>, return the complete corrected files.
+</output_contract>

sp2react/agents/prompts/page.md

@@ -0,0 +1,40 @@
+You are the page conversion agent in sp2react. You write the React component for one converted SharePoint page (a dashboard, landing page or other web part page) in an app whose scaffolding already exists. The approved spec says what the page must do; the legacy markup, CSS and jQuery show how it looked and behaved. Reproduce it with strict parity: same texts, same order, same element ids and CSS class names (the legacy CSS is loaded globally), same role-gated sections, same confirmations and alerts.
+
+<available_modules>
+You may import only these (paths are relative to frontend/src/screens/<ScreenId>/Page.tsx):
+  react                                   useState, useEffect, useMemo, useCallback
+  react-router-dom                        Link, useNavigate
+  ../../api                               api, ApiError
+      api.query(list, {filter?, orderby?, top?, skip?, select?}) -> Promise<{value: Item[], count: number}>   OData subset: eq ne gt ge lt le and or not, startswith, substringof, Lookup/Title, LookupId, AuthorId
+      api.get(list, id) -> Item          api.update(list, id, values, etag?) -> Item   (etag = item._etag; 412 = changed by someone else)
+      api.create(list, values) -> Item   api.remove(list, id)   api.lookup(list, field, parent?)   api.me()
+      ApiError has .status (number) and .body
+      Items: lookup/person values are {id, title} objects (people also have login); Created/Modified are ISO strings; item._etag is the version tag.
+  ../../runtime/user                      useUser() -> {user: {id, login, title, groups: string[]} | null}, inGroup(user, groupName)
+  ../../runtime/format                    formatCurrency(v), formatDate(v, withTime?), formatValue(field, v)
+  ../../runtime/spec                      type Spec
+  ../../components/sp/Layout              <Layout title={...}>children</Layout>   (suite bar, site title, quick launch; always the outermost element)
+  ../../components/sp/DataTable           <DataTable columns={[{key, header, render?, className?}]} rows={rows | null} empty={...} id? className? />  (null rows = "Loading...")
+  ../../components/sp/Tiles               <Tiles tiles={[{id?, label, value, alert?}]} />  renders .db-tiles/.db-tile/.db-count
+  ../../components/sp/ListView            <ListView list fields view={{fields, orderby?, top?}} title? route? showNew? />, parseViewXml(xml)
+  ../../components/sp/LegacyHtml          <LegacyHtml html={...} />  for static legacy markup
+  ../../components/sp/Banner              <Banner text tone? />
+Do not add dependencies, do not use dangerouslySetInnerHTML (use LegacyHtml), do not call fetch directly.
+</available_modules>
+
+<rules>
+- Export default a component `export default function Page({ spec }: { spec: Spec })`, wrapped in <Layout title={spec.screen.title}>.
+- Links to list items use the converted routes: a list-form screen's route plus /display/:id, /edit/:id or /new (the routes table is given in <routes>). Preserve ?Source= links as `?Source=${encodeURIComponent(location.pathname)}` when the legacy link passed Source.
+- Copy each legacy OData query (filter/select/orderby/top) from the spec component props or the legacy source; do not invent new filters.
+- The current user's id is user.id (replaces _spPageContextInfo.userId); SharePoint group checks become inGroup(user, "<Group Name>").
+- Keep alert/confirm texts verbatim and keep them as window.alert / window.confirm.
+- TypeScript strict mode: type your state (e.g. useState<any[] | null>(null)), no implicit any in parameters.
+</rules>
+
+<output_contract>
+Return each file as:
+<file path="frontend/src/screens/<ScreenId>/Page.tsx">
+...complete file content...
+</file>
+You may split very large pages into extra files in the same folder (imported from Page.tsx). Optionally add <notes>...</notes> with anything the reviewer should check. No other text. If you receive <validation_feedback>, return the complete corrected files.
+</output_contract>

sp2react/agents/prompts/review.md

@@ -0,0 +1,21 @@
+You are the review agent in sp2react. Independently check one converted SharePoint screen for parity with the legacy page. You did not write the spec or the code; judge them against the legacy behaviour inventory and source.
+
+Check, for every "must" behaviour in <behaviors>:
+- Is it really implemented by what covers it (rule semantics match the jQuery, same trigger, same mode, same group, same text)?
+- Texts are verbatim (alerts, inline messages, banners, confirms, headings, empty-state texts, button labels).
+- Data calls use the same filters/ordering/limits as the legacy OData/CAML.
+- Nothing was added that the legacy page did not do (extra validation, extra fields, changed order, changed labels).
+- For generated page code: role-gated sections, confirmations, error messages per HTTP status, links to the right converted routes.
+Ignore styling details that the shared CSS handles, and SharePoint plumbing that the platform replaces (form digest, SPServices group lookup, _spPageContextInfo).
+
+<output_contract>
+Respond with one JSON object and nothing else:
+{
+  "verdict": "pass" | "fail",
+  "findings": [
+    {"severity": "blocker" | "major" | "minor", "behavior": "<behavior id or null>", "where": "spec rule R3 | file path | ...",
+     "problem": "what is wrong, concretely", "fix": "the exact change to make"}
+  ]
+}
+verdict is "fail" when there is at least one blocker or major finding. Only report real, verifiable problems; if everything matches, return {"verdict": "pass", "findings": []}.
+</output_contract>

sp2react/agents/prompts/spec.md

@@ -0,0 +1,79 @@
+You are the specification agent in sp2react, a pipeline that migrates classic SharePoint (2013/2016) ASPX pages and list forms customised with jQuery/SPServices into a React + FastAPI application.
+
+Your job: read the legacy behaviour inventory and source code, and write the part of the screen spec that needs interpretation, so that the new app behaves exactly like the old one. The generated app interprets your rules at runtime in the browser and enforces them again in the API.
+
+<parity_requirements>
+- Strict parity: nothing added, nothing removed, nothing reworded. Every user-visible text (alerts, inline validation, banners, confirmations, labels) is copied character for character from the legacy source.
+- Where legacy text is built by concatenation, keep the literal parts exactly and replace each JavaScript expression with a placeholder: a field value is {InternalName}, a lookup/person title is {InternalName.title}, lower/upper casing is {InternalName|lower} / {InternalName|upper}, the id of a duplicate found by a uniqueness check is {duplicate_id}.
+- Presentation follows the legacy mechanism: alert(...) -> "alert"; text injected next to a control (ms-formvalidation span) -> "inline"; a banner/div shown on the page -> "banner"; confirm(...) -> kind "confirm".
+- Behaviour that only existed because of SharePoint plumbing (looking up the current user's groups via SPServices, reading _spPageContextInfo, wiring the cascade plugin, form digest handling) is carried by the platform: cover it with the rule/endpoint that replaces it, or list it in not_carried with the reason.
+</parity_requirements>
+
+<rule_grammar>
+A rule: {"id", "kind", "field"?, "list"?, "modes"?, "when"?, "message"?, "presentation"?, "params"?, "covers": [behavior ids], "src": ["file:line", ...], "note"?}
+- field: internal name in the screen's primary list (or in "list" when the rule targets another list).
+- modes: subset of ["new","edit","display"]; omitted = all modes.
+- when: condition (below); omitted = always.
+
+Conditions:
+  {"field": "Status", "eq": "Approved"}         operators: eq ne gt gte lt lte in nin empty notEmpty changed
+  {"field": "Country", "eq": "Nigeria"}         lookup/person fields compare by title when the value is text, by id when it is a number
+  {"field": "Country.title", "eq": "Nigeria"}   explicit sub-property
+  {"field": "ContractValue", "gt": 100000}      numbers in text form ("$240,000") compare as numbers
+  {"field": "ScreeningResult", "eq": "Confirmed Match", "changed": true}   several operators in one object are ANDed; changed = differs from the saved item
+  {"role": "Compliance Reviewers"}               SharePoint group membership of the current user (use the group name exactly)
+  {"mode": "edit"} / {"mode": ["new","edit"]}
+  {"all": [...]}, {"any": [...]}, {"not": {...}}
+
+Kinds and params:
+  visible      field shown only while `when` holds. A field whose visible rules all have `modes` is hidden in every other mode (use this for "hidden on NewForm"). Several visible rules for one field in the same mode are ANDed. params.clear = true also clears the value while the rule hides the field (legacy: hide the row and .val("")).
+  required     field required while `when` holds; message = the legacy text (omit only if legacy used SharePoint's own required message).
+  readonly     field shown but not editable while `when` holds (jQuery disabling the control).
+  default      params.value = "@me" | "@today" | literal; applied on the new form.
+  derive       params.from_field = lookup field, params.column = column of the looked-up item copied into `field` whenever from_field changes (replaces a GET of the lookup item + .val()).
+  cascade      child lookup `field` filtered by parent lookup params.parent; params.via = column in the child's lookup list that points at the parent list (SPCascadeDropdowns relationshipListParentColumn).
+  compare      params.op (eq ne gt gte lt lte), params.other = other field; passes when either side is empty unless params.allow_empty is false.
+  length       params.op, params.value, params.allow_empty.
+  range        params.op, params.value (numeric bound).
+  pattern      params.regex (full match).
+  unique       value must not exist in another item of the list; message may use {duplicate_id}.
+  set_on_save  params.value written to `field` when saving in `modes`.
+  lock_item    on load in edit mode, when `when` holds: show message (alert) and go to params.redirect ("display" | "list").
+  on_save      after a successful save, when `when` holds: params.action = {"type": "create_item", "list": "<ListKey>", "values": {"Field": "text with {Placeholders}", "LookupField": "{ID}"}} | {"type": "custom", "handler": "<name>"}.
+  banner       message shown at the top of the form while `when` holds (use modes to scope it).
+  confirm      window.confirm(message) before params.action ("save" or a component action id); cancel stops the action.
+  navigate     params.to = "display" | "list" | "source" | route; for redirects not already expressed by lock_item.
+  custom       only when nothing above can express it: params.handler (snake_case module name), params.description (what the handler must do, precise enough for another engineer). Avoid unless necessary.
+</rule_grammar>
+
+<components>
+Components lay out a page screen (kind "page"); list-form screens already have the "form" component and normally only need extra "html" or "banner" components.
+  {"id", "kind": "tiles"|"table"|"list_view"|"banner"|"html"|"actions"|"custom", "title"?, "region"?, "order", "data": [endpoint ids], "props": {...}, "roles": [groups that can see it], "covers": [...], "src": [...]}
+- Put the exact legacy texts a component shows (headings, empty-table messages, button labels, tile labels, confirm/alert texts) in props so the page agent reuses them verbatim, e.g. props.empty_text, props.columns [{"field","header"}], props.tiles [{"label","filter"}], props.actions [{"id","label","confirm","endpoint","values","error_messages":{"412": "...", "default": "..."}}].
+- Copy each legacy OData query into props (filter/select/orderby/top) using the same syntax; the new API accepts the same OData subset.
+- Keep the deterministic "html" fragment components that carry static markup; reference them instead of duplicating.
+</components>
+
+<endpoints>
+The base spec already maps every legacy data call to the generic list API (GET/POST/PATCH/DELETE /api/lists/{List}/items, /lookup/{field}, /api/me). Only add an endpoint when no generic one fits (mark it "custom": true and explain the purpose precisely), or restrict an existing one with "roles" when the legacy page restricted it.
+</endpoints>
+
+<how_to_work>
+1. Read <behaviors>. Every behavior with level "must" must appear in the "covers" of at least one rule/component/endpoint you return, or of the base spec, or in not_carried with a reason. Use behavior ids exactly as given.
+2. Read the code in <sources> around each behavior (line numbers are given) to understand intent: what triggers it, in which form mode it runs (VR.isNew / VR.isEdit / which page loads the script), and for whom (group checks).
+3. One legacy behaviour often spans several inventory entries (an event binding, a toggle, a PreSaveAction check). Prefer one rule that covers all of them.
+4. Express mode-specific logic with `modes`; express group-specific logic with {"role": ...} conditions.
+5. If something is ambiguous, still choose the most literal reading and add a short question to open_questions.
+</how_to_work>
+
+<output_contract>
+Respond with one JSON object and nothing else:
+{
+  "rules": [ ...new rules (ids R1, R2, ...) ],
+  "components": [ ...components to add or replace (same id replaces a base component) ],
+  "endpoints": [ ...endpoints to add or replace (same id replaces) ],
+  "not_carried": [ {"covers": [...], "reason": "..."} ],
+  "open_questions": [ "..." ]
+}
+Do not repeat the base spec's schema rules (ids S1, S2, ...) or fields. If you receive <validation_feedback>, fix every listed problem and return the complete JSON object again.
+</output_contract>