sp2react 0.1.0__py3-none-any.whl

sp2react/scaffold/templates/frontend/src/styles/sp.css

@@ -0,0 +1,95 @@
+/* SharePoint 2013/2016 look (corev15). Legacy site CSS is appended in legacy.css. */
+:root {
+  --sp-blue: #0072c6;
+  --sp-text: #444;
+  --sp-heading: #262626;
+  --sp-border: #ababab;
+  --sp-subtle: #c6c6c6;
+  --sp-error: #bf0000;
+  --sp-accent: #0072c6;
+}
+* { box-sizing: border-box; }
+body { margin: 0; font-family: "Segoe UI", "Segoe", Tahoma, Helvetica, Arial, sans-serif; font-size: 13px; color: var(--sp-text); background: #fff; }
+a { color: var(--sp-blue); text-decoration: none; }
+a:hover { text-decoration: underline; }
+
+.sp-suitebar { height: 30px; background: var(--sp-blue); color: #fff; display: flex; align-items: center; justify-content: space-between; padding: 0 12px; }
+.sp-brand { font-size: 17px; font-weight: 300; }
+.sp-suite-right { display: flex; gap: 16px; align-items: center; font-size: 13px; }
+.sp-devuser select { font: inherit; font-size: 12px; margin-left: 4px; }
+.sp-titlerow { padding: 18px 20px 10px 200px; }
+.sp-sitetitle a { color: var(--sp-blue); font-size: 12px; }
+.sp-pagetitle { font-size: 2em; font-weight: 300; color: var(--sp-heading); margin: 2px 0 0; }
+.sp-body { display: flex; }
+.sp-quicklaunch { width: 180px; flex: 0 0 180px; padding: 6px 0 0 20px; }
+.sp-quicklaunch ul { list-style: none; margin: 0; padding: 0; }
+.sp-quicklaunch li a { display: block; padding: 6px 8px; font-size: 15px; color: var(--sp-text); }
+.sp-quicklaunch li.selected a { background: #e5e5e5; }
+.sp-content { flex: 1; min-width: 0; padding: 4px 20px 40px; overflow-x: auto; }
+@media (max-width: 760px) {
+  .sp-titlerow { padding-left: 16px; }
+  .sp-quicklaunch { display: none; }
+  .sp-content { padding: 4px 16px 32px; }
+  .ms-formtable td.ms-formlabel { width: auto !important; display: block; }
+  .ms-formtable td.ms-formbody { display: block; width: auto; }
+  .ms-formtable, .ms-formtable > tbody, .ms-formtable > tbody > tr { display: block; width: 100%; }
+  .ms-formtable td { padding-right: 0; }
+}
+
+/* list forms */
+.ms-formtable { border-collapse: collapse; margin: 8px 0 12px; }
+.ms-formtable td { vertical-align: top; padding: 6px 8px 8px 0; }
+.ms-formlabel { width: 165px; }
+.ms-standardheader { font-size: 13px; font-weight: 400; color: var(--sp-heading); margin: 2px 0; }
+.ms-accentText { color: var(--sp-error); }
+.ms-formbody { width: 450px; }
+.ms-long { width: 386px; max-width: 100%; }
+.ms-formbody input[type="text"], .ms-formbody input[type="date"], .ms-formbody input[type="datetime-local"], .ms-formbody select, .ms-formbody textarea {
+  font: inherit; color: var(--sp-text); border: 1px solid var(--sp-border); padding: 3px 4px; background: #fff;
+}
+.ms-formbody input:disabled, .ms-formbody select:disabled, .ms-formbody textarea:disabled { background: #fdfdfd; color: #b1b1b1; border-color: #e1e1e1; }
+.ms-formbody select { min-width: 120px; }
+.ms-metadata { color: #777; font-size: 12px; margin-top: 2px; }
+.ms-formvalidation { color: var(--sp-error); font-size: 12px; }
+.sp-currency { margin-right: 3px; }
+.sp-radios label, .sp-checks label { display: block; }
+.sp-toolbar { display: flex; gap: 8px; justify-content: flex-end; width: 615px; max-width: 100%; margin: 6px 0; }
+.ms-ButtonHeightWidth { font: inherit; min-width: 80px; height: 30px; padding: 0 12px; border: 1px solid var(--sp-border); background: #fdfdfd; color: var(--sp-text); cursor: pointer; }
+.ms-ButtonHeightWidth:hover { border-color: #92c0e0; background: #e6f2fa; }
+.ms-ButtonHeightWidth:disabled { color: #b1b1b1; cursor: default; }
+.sp-itemmeta { color: #777; font-size: 12px; margin: 10px 0; }
+.sp-link { background: none; border: 0; color: var(--sp-blue); cursor: pointer; font: inherit; margin-left: 8px; }
+.sp-loading { color: #777; padding: 12px 0; }
+
+/* people picker */
+.sp-peoplepicker { position: relative; border: 1px solid var(--sp-border); min-height: 26px; width: 386px; max-width: 100%; padding: 2px 4px; display: flex; flex-wrap: wrap; gap: 4px; align-items: center; background: #fff; }
+.sp-peoplepicker.is-disabled { background: #fdfdfd; border-color: #e1e1e1; }
+.sp-peoplepicker input { border: 0 !important; outline: none; flex: 1; min-width: 120px; padding: 2px; }
+.ms-entity-resolved { background: #f4f4f4; padding: 1px 4px; color: var(--sp-heading); }
+.ms-entity-resolved button { border: 0; background: none; cursor: pointer; color: #777; margin-left: 2px; }
+.sp-picker-hits { position: absolute; top: 100%; left: -1px; right: -1px; z-index: 5; list-style: none; margin: 0; padding: 0; background: #fff; border: 1px solid var(--sp-border); }
+.sp-picker-hits li { padding: 6px 8px; cursor: pointer; }
+.sp-picker-hits li:hover { background: #e6f2fa; }
+.sp-muted { color: #777; }
+
+/* banners */
+.sp-banner { padding: 8px 12px; margin: 0 0 12px; font-weight: 600; }
+.sp-banner-error { background: #fde7e9; border: 1px solid #a80000; color: #a80000; }
+.sp-banner-info { background: #f4f4f4; border-left: 4px solid var(--sp-blue); font-weight: 400; }
+
+/* list views */
+.ms-webpart-titleText { font-size: 1.46em; font-weight: 300; color: var(--sp-heading); margin: 16px 0 8px; }
+.sp-newitem { margin: 8px 0; font-size: 15px; }
+.ms-listviewtable { border-collapse: collapse; width: 100%; }
+.ms-listviewtable th { text-align: left; font-weight: 400; color: #777; border-bottom: 1px solid var(--sp-subtle); padding: 6px 8px; white-space: nowrap; }
+.ms-listviewtable td { padding: 6px 8px; border-bottom: 1px solid #eaeaea; }
+.nobr { white-space: nowrap; }
+
+/* Narrow screens: stack legacy layout tables, wrap tile rows, let wide tables scroll inside their own box */
+@media (max-width: 760px) {
+  .sp-content table[width="100%"] > tbody > tr > td { display: block; width: auto !important; }
+  .db-tiles { flex-wrap: wrap; }
+  .db-tile { min-width: calc(50% - 6px); }
+  .db-table, .ms-listviewtable { display: block; overflow-x: auto; }
+  .sp-content { padding-right: max(16px, env(safe-area-inset-right, 0px)); }
+}

sp2react/scaffold/templates/frontend/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

sp2react/scaffold/templates/frontend/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "lib": ["ES2020", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "jsx": "react-jsx",
+    "strict": true,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "skipLibCheck": true,
+    "allowImportingTsExtensions": false,
+    "noEmit": true
+  },
+  "include": ["src"]
+}

sp2react/scaffold/templates/frontend/vite.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from "vite";
+import react from "@vitejs/plugin-react";
+
+export default defineConfig({
+  plugins: [react()],
+  server: { port: 5173, proxy: { "/api": "http://localhost:8000" } },
+});

sp2react/sharepoint/client.py

@@ -0,0 +1,210 @@
+"""Read-only SharePoint REST client (SharePoint Server 2013/2016/2019/SE and SharePoint Online).
+
+Only GET requests are ever issued: the extractor must never change the source site.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+from urllib.parse import quote, urlsplit
+
+import requests
+
+from ..config import SharePointConfig, secret
+
+log = logging.getLogger(__name__)
+
+
+class SharePointError(RuntimeError):
+    def __init__(self, message: str, status: Optional[int] = None):
+        super().__init__(message)
+        self.status = status
+
+
+def odata_quote(value: str) -> str:
+    """Escape a value for use inside '...' in a SharePoint REST path."""
+    return quote(value.replace("'", "''"), safe="/:")
+
+
+def unwrap(data: Any) -> Any:
+    """Strip odata=verbose wrappers: {"d": {...}} and {"results": [...]}."""
+    if isinstance(data, dict) and "d" in data and len(data) == 1:
+        data = data["d"]
+    if isinstance(data, dict) and "results" in data and isinstance(data["results"], list):
+        return data["results"]
+    return data
+
+
+class ItemReadBlocked(SharePointError):
+    pass
+
+
+class SharePointClient:
+    """`policy` limits which hosts may be contacted; `item_reads=False` forbids reading list items, attachments,
+    user profiles and rendered pages (anything that carries business data), leaving only code and schema."""
+
+    def __init__(self, cfg: SharePointConfig, session: Optional[requests.Session] = None, policy=None, item_reads: bool = True):
+        self.cfg = cfg
+        self.item_reads = item_reads
+        self.site_url = cfg.site_url
+        parts = urlsplit(cfg.site_url)
+        self.origin = f"{parts.scheme}://{parts.netloc}"
+        self.server_relative = parts.path.rstrip("/") or "/"
+        self.session = session or requests.Session()
+        self.session.verify = cfg.verify_ssl
+        self.session.headers.update({"Accept": "application/json;odata=verbose", "User-Agent": "sp2react/0.1 (read-only)"})
+        self.session.headers.update(cfg.extra_headers)
+        if policy is not None:
+            policy.guard(self.session)
+        send = self.session.send
+
+        def get_only(request, **kwargs):          # the extractor must never change the source site
+            if request.method not in ("GET", "HEAD"):
+                raise SharePointError(f"sp2react only reads from SharePoint; refused {request.method} {request.url}")
+            return send(request, **kwargs)
+
+        self.session.send = get_only  # type: ignore[method-assign]
+        self._configure_auth()
+
+    # ------------------------------------------------------------------ auth
+    def _configure_auth(self) -> None:
+        mode = self.cfg.auth
+        if mode == "ntlm":
+            try:
+                from requests_ntlm import HttpNtlmAuth  # type: ignore
+            except ImportError as e:  # pragma: no cover - depends on optional extra
+                raise SharePointError("NTLM auth needs the extra: pip install 'sp2react[ntlm]'") from e
+            self.session.auth = HttpNtlmAuth(
+                secret(self.cfg.username_env, "SharePoint user name (DOMAIN\\user)"),
+                secret(self.cfg.password_env, "SharePoint password"),
+            )
+        elif mode == "cookie":
+            self.session.headers["Cookie"] = secret(self.cfg.cookie_env, "SharePoint session cookie")
+        elif mode == "bearer":
+            self.session.headers["Authorization"] = "Bearer " + secret(self.cfg.token_env, "SharePoint bearer token")
+
+    # --------------------------------------------------------------- helpers
+    def absolute(self, url_or_path: str) -> str:
+        if url_or_path.startswith(("http://", "https://")):
+            return url_or_path
+        if url_or_path.startswith("/"):
+            return self.origin + url_or_path
+        return f"{self.site_url}/{url_or_path}"
+
+    def site_relative(self, server_relative_url: str) -> str:
+        """'/sites/x/Lists/A/NewForm.aspx' -> 'Lists/A/NewForm.aspx'."""
+        prefix = self.server_relative.rstrip("/") + "/"
+        if server_relative_url.lower().startswith(prefix.lower()):
+            return server_relative_url[len(prefix):]
+        return server_relative_url.lstrip("/")
+
+    def server_relative_url(self, path: str) -> str:
+        if path.startswith(("http://", "https://")):
+            return urlsplit(path).path
+        if path.startswith("/"):
+            return path
+        return self.server_relative.rstrip("/") + "/" + path
+
+    def _data(self, what: str) -> None:
+        if not self.item_reads:
+            raise ItemReadBlocked(f"Reading {what} from SharePoint is disabled (data_source=database / security.sharepoint_item_reads=false).")
+
+    def _get(self, url: str, accept: Optional[str] = None, params: Optional[dict] = None) -> requests.Response:
+        headers = {"Accept": accept} if accept else None
+        try:
+            r = self.session.get(self.absolute(url), headers=headers, params=params, timeout=self.cfg.timeout)
+        except requests.RequestException as e:
+            raise SharePointError(f"GET {url} failed: {e}") from e
+        if r.status_code >= 400:
+            raise SharePointError(f"GET {url} -> HTTP {r.status_code}: {r.text[:300]}", r.status_code)
+        return r
+
+    def api(self, path: str, params: Optional[dict] = None) -> Any:
+        r = self._get(f"{self.site_url}/_api/{path.lstrip('/')}", params=params)
+        return unwrap(r.json())
+
+    def text(self, url: str, accept: str = "*/*") -> str:
+        r = self._get(url, accept=accept)
+        r.encoding = r.encoding or "utf-8"
+        return r.text
+
+    # ------------------------------------------------------------ endpoints
+    def web(self) -> dict:
+        return self.api("web", {"$select": "Title,Url,ServerRelativeUrl,UIVersion,Language"})
+
+    def lists(self) -> list[dict]:
+        return self.api("web/lists", {
+            "$select": "Id,Title,EntityTypeName,BaseTemplate,BaseType,Hidden,EnableAttachments,EnableVersioning,ItemCount,"
+                       "ValidationFormula,ValidationMessage,RootFolder/ServerRelativeUrl",
+            "$expand": "RootFolder",
+            "$filter": "Hidden eq false",
+        })
+
+    def list_forms(self, list_id: str) -> list[dict]:
+        return self.api(f"web/lists(guid'{list_id.strip('{}')}')/forms", {"$select": "ServerRelativeUrl,FormType"})
+
+    def list_views(self, list_id: str) -> list[dict]:
+        return self.api(f"web/lists(guid'{list_id.strip('{}')}')/views", {
+            "$select": "Title,ServerRelativeUrl,DefaultView,ViewQuery,RowLimit,Hidden",
+            "$expand": "ViewFields",
+        })
+
+    def fields(self, list_id: str) -> list[dict]:
+        return self.api(f"web/lists(guid'{list_id.strip('{}')}')/fields")
+
+    def items(self, list_id: str, top: int) -> list[dict]:
+        self._data("list items")
+        return self.api(f"web/lists(guid'{list_id.strip('{}')}')/items", {"$top": str(top)})
+
+    def quick_launch(self) -> list[dict]:
+        return self.api("web/navigation/quicklaunch", {"$select": "Title,Url"})
+
+    def all_items(self, list_id: str, page_size: int = 1000):
+        """Every item of a list, following SharePoint's __next paging links (works past the 5,000 view threshold
+        because it pages by ID). Yields raw odata=verbose items, attachments listed under AttachmentFiles."""
+        self._data("list items")
+        url = f"{self.site_url}/_api/web/lists(guid'{list_id.strip('{}')}')/items"
+        params: Optional[dict] = {"$top": str(page_size), "$orderby": "ID", "$expand": "AttachmentFiles"}
+        while url:
+            r = self._get(url, params=params)
+            data = r.json().get("d", {})
+            for it in data.get("results", []):
+                yield it
+            url, params = data.get("__next"), None
+
+    def file_bytes(self, server_relative_url: str) -> bytes:
+        self._data("attachments")
+        url = self.server_relative_url(server_relative_url)
+        return self._get(f"{self.site_url}/_api/web/getfilebyserverrelativeurl('{odata_quote(url)}')/$value", accept="*/*").content
+
+    def site_users(self) -> list[dict]:
+        self._data("site users")
+        return self.api("web/siteusers", {"$select": "Id,LoginName,Title,Email,PrincipalType"})
+
+    def site_groups(self) -> list[dict]:
+        if not self.item_reads:      # group names only; membership (people) stays in SharePoint
+            return self.api("web/sitegroups", {"$select": "Id,Title"})
+        return self.api("web/sitegroups", {"$select": "Id,Title,Users/LoginName,Users/Title", "$expand": "Users"})
+
+    def file_text(self, server_relative_url: str) -> str:
+        """Raw stored content of a file (the .aspx source, not the rendered page)."""
+        url = self.server_relative_url(server_relative_url)
+        return self.text(f"{self.site_url}/_api/web/getfilebyserverrelativeurl('{odata_quote(url)}')/$value")
+
+    def web_parts(self, server_relative_url: str) -> list[dict]:
+        """Web parts added through the browser live in the content DB, not the .aspx; this is how to see them."""
+        url = self.server_relative_url(server_relative_url)
+        return self.api(
+            f"web/getfilebyserverrelativeurl('{odata_quote(url)}')/getlimitedwebpartmanager(1)/webparts",
+            {"$expand": "WebPart/Properties"},
+        )
+
+    def folder_files(self, server_relative_folder: str) -> list[dict]:
+        url = self.server_relative_url(server_relative_folder)
+        return self.api(f"web/getfolderbyserverrelativeurl('{odata_quote(url)}')/files", {"$select": "Name,ServerRelativeUrl,TimeLastModified"})
+
+    def rendered(self, server_relative_url: str) -> str:
+        """The page as a browser receives it (shows runtime-injected scripts). Can contain list data, so it is
+        only fetched when item reads are allowed."""
+        self._data("rendered pages")
+        return self.text(self.server_relative_url(server_relative_url), accept="text/html")

sp2react/sharepoint/extract.py

@@ -0,0 +1,217 @@
+"""Stage 2 (scan and crawl): pull everything the conversion needs from the live site, read-only.
+
+Workspace layout written under <workspace>/legacy (identical to tests/fixtures/legacy-site):
+  site.json, siteusers.json
+  lists/<ListUrlName>/fields.json, items.json
+  pages/<site-relative path>.aspx              raw stored source
+  pages/<site-relative path>.aspx.webparts.json browser-added web parts (web part manager)
+  rendered/<site-relative path without .aspx>.html  what the browser receives
+  files/<site-relative path>                   custom scripts, css, CEWP html (SiteAssets, Style Library...)
+  manifest.json                                what was fetched and what failed
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Iterable, Optional
+
+from ..analysis.aspx import asset_refs, parse_page, resolve_tokens
+from .client import ItemReadBlocked, SharePointClient, SharePointError, unwrap
+
+log = logging.getLogger(__name__)
+
+FORM_TYPES = {8: "New", 6: "Edit", 4: "Display"}
+SKIP_PREFIXES = ("/_layouts/", "/_vti_bin/", "/_catalogs/masterpage/", "/scriptresource.axd", "/webresource.axd")
+
+
+def _write(path: Path, content) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if isinstance(content, (dict, list)):
+        path.write_text(json.dumps(content, indent=2, ensure_ascii=False), encoding="utf-8")
+    else:
+        path.write_text(content, encoding="utf-8")
+
+
+def _list_folder(list_entry: dict) -> str:
+    return list_entry["url"].rstrip("/").split("/")[-1]
+
+
+class Extractor:
+    def __init__(self, client: SharePointClient, workspace: Path, progress=None):
+        self.sp = client
+        self.root = workspace / "legacy"
+        self.progress = progress or (lambda msg: log.info(msg))
+        self.manifest: dict = {"site": client.site_url, "pages": [], "lists": [], "files": [], "errors": []}
+
+    def _err(self, what: str, e: Exception) -> None:
+        self.manifest["errors"].append({"what": what, "error": str(e)[:400]})
+        log.warning("%s: %s", what, e)
+
+    # ----------------------------------------------------------- site/lists
+    def site(self) -> dict:
+        web = self.sp.web()
+        site = {"url": self.sp.site_url, "serverRelativeUrl": web.get("ServerRelativeUrl", self.sp.server_relative),
+                "title": web.get("Title"), "uiVersion": web.get("UIVersion"), "lists": [], "groups": [], "users": []}
+        for l in self.sp.lists():
+            if l.get("BaseType", 0) not in (0, None) or l.get("BaseTemplate") in (101, 109, 119, 851, 700):
+                continue
+            root = (l.get("RootFolder") or {}).get("ServerRelativeUrl", "")
+            entry = {"title": l["Title"], "url": self.sp.site_relative(root), "id": "{" + l["Id"].strip("{}") + "}",
+                     "entityTypeName": l.get("EntityTypeName"), "enableAttachments": l.get("EnableAttachments", False),
+                     "enableVersioning": l.get("EnableVersioning", False), "itemCount": l.get("ItemCount"),
+                     "validationFormula": l.get("ValidationFormula") or None, "validationMessage": l.get("ValidationMessage") or None,
+                     "forms": {}, "views": {}}
+            try:
+                for f in self.sp.list_forms(l["Id"]):
+                    mode = FORM_TYPES.get(f.get("FormType"))
+                    if mode and mode not in entry["forms"]:
+                        entry["forms"][mode] = f["ServerRelativeUrl"].split("/")[-1]
+            except SharePointError as e:
+                self._err(f"forms of {l['Title']}", e)
+            try:
+                for v in self.sp.list_views(l["Id"]):
+                    if v.get("Hidden"):
+                        continue
+                    vf = v.get("ViewFields") or {}
+                    fields = unwrap(vf.get("Items")) or []
+                    name = (v.get("ServerRelativeUrl") or v.get("Title", "View")).split("/")[-1].replace(".aspx", "")
+                    entry["views"][name] = {"title": v.get("Title"), "fields": fields, "query": v.get("ViewQuery"),
+                                            "rowLimit": v.get("RowLimit"), "default": v.get("DefaultView", False)}
+            except SharePointError as e:
+                self._err(f"views of {l['Title']}", e)
+            site["lists"].append(entry)
+        return site
+
+    def list_data(self, site: dict, sample_items: int) -> None:
+        for entry in site["lists"]:
+            folder = self.root / "lists" / _list_folder(entry)
+            self.progress(f"list {entry['title']}: fields and sample items")
+            try:
+                _write(folder / "fields.json", {"d": {"results": self.sp.fields(entry["id"])}})
+            except SharePointError as e:
+                self._err(f"fields of {entry['title']}", e)
+                continue
+            if not self.sp.item_reads:
+                _write(folder / "items.json", {"d": {"results": []}, "note": "list items are not read: data_source=database"})
+            else:
+                try:
+                    _write(folder / "items.json", {"d": {"results": self.sp.items(entry["id"], sample_items)}})
+                except SharePointError as e:
+                    self._err(f"items of {entry['title']}", e)
+            self.manifest["lists"].append(entry["title"])
+
+    def principals(self, site: dict) -> None:
+        if not self.sp.item_reads:
+            site["users"] = []
+        try:
+            users = self.sp.site_users() if self.sp.item_reads else []
+            _write(self.root / "siteusers.json", {"d": {"results": users}})
+            site["users"] = [{"login": re.sub(r"^i:0#\.\w\|", "", u.get("LoginName", "")), "title": u.get("Title"), "email": u.get("Email")}
+                             for u in users if u.get("PrincipalType", 1) == 1]
+        except SharePointError as e:
+            self._err("site users", e)
+        try:
+            site["groups"] = [{"name": g["Title"], "members": [re.sub(r"^i:0#\.\w\|", "", u.get("LoginName", "")) for u in (unwrap(g.get("Users")) or [])] if isinstance(g.get("Users"), dict) else []}
+                              for g in self.sp.site_groups()]
+        except SharePointError as e:
+            self._err("site groups (needs permission to enumerate groups)", e)
+
+    # ---------------------------------------------------------------- pages
+    def discover_pages(self, site: dict) -> list[str]:
+        pages = []
+        for entry in site["lists"]:
+            for form in entry.get("forms", {}).values():
+                pages.append(f"{entry['url']}/{form}")
+        try:
+            for f in self.sp.folder_files("SitePages"):
+                if f["Name"].lower().endswith(".aspx"):
+                    pages.append(self.sp.site_relative(f["ServerRelativeUrl"]))
+        except SharePointError as e:
+            self._err("SitePages library", e)
+        return pages
+
+    def page(self, page: str) -> Optional[str]:
+        rel = self.sp.site_relative(self.sp.server_relative_url(page))
+        self.progress(f"page {rel}")
+        try:
+            source = self.sp.file_text(rel)
+        except SharePointError as e:
+            self._err(f"source of {rel}", e)
+            return None
+        _write(self.root / "pages" / rel, source)
+        wps = None
+        try:
+            wps = self.sp.web_parts(rel)
+            _write(self.root / "pages" / (rel + ".webparts.json"), {"d": {"results": wps}})
+        except SharePointError as e:
+            self._err(f"web parts of {rel}", e)
+        try:
+            if self.sp.item_reads:
+                _write(self.root / "rendered" / (re.sub(r"\.aspx$", "", rel, flags=re.I) + ".html"), self.sp.rendered(rel))
+        except SharePointError as e:
+            self._err(f"rendered {rel}", e)
+        self.manifest["pages"].append(rel)
+        return rel
+
+    # --------------------------------------------------------------- assets
+    def assets(self, pages: Iterable[str]) -> None:
+        queue: list[str] = []
+        for rel in pages:
+            src = (self.root / "pages" / rel).read_text(encoding="utf-8", errors="replace")
+            wpf = self.root / "pages" / (rel + ".webparts.json")
+            wpj = json.loads(wpf.read_text(encoding="utf-8"))["d"]["results"] if wpf.exists() else None
+            queue += asset_refs(parse_page(rel, src, wpj))
+            rendered = self.root / "rendered" / (re.sub(r"\.aspx$", "", rel, flags=re.I) + ".html")
+            if rendered.exists():
+                from ..analysis.aspx import scan_html
+                s, _, css, _ = scan_html(rendered.read_text(encoding="utf-8", errors="replace"))
+                queue += s + css
+        done = set()
+        while queue:
+            ref = resolve_tokens(queue.pop(0), self.sp.server_relative)
+            url = ref.split("?")[0].split("#")[0]
+            if url.startswith(("http://", "https://")):
+                if not url.lower().startswith(self.sp.origin.lower()):
+                    continue                      # CDN / external: recorded by the inventory, not downloaded
+                url = url[len(self.sp.origin):]
+            if not url.startswith("/"):
+                url = self.sp.server_relative_url(url)
+            if url.lower().startswith(SKIP_PREFIXES) or url in done:
+                continue
+            if not url.lower().startswith(self.sp.server_relative.lower().rstrip("/") + "/"):
+                continue                          # outside this site (another site collection): not ours to export
+            done.add(url)
+            rel = self.sp.site_relative(url)
+            try:
+                text = self.sp.file_text(url)
+            except SharePointError as e:
+                self._err(f"asset {rel}", e)
+                continue
+            _write(self.root / "files" / rel, text)
+            self.manifest["files"].append(rel)
+            if rel.lower().endswith((".html", ".htm", ".txt")):  # CEWP content links can pull in more scripts
+                from ..analysis.aspx import scan_html
+                s, _, css, _ = scan_html(text)
+                queue += s + css
+
+    # ------------------------------------------------------------------ run
+    def run(self, pages: Optional[list[str]] = None, sample_items: int = 200) -> dict:
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.progress("site and lists")
+        site = self.site()
+        self.principals(site)
+        try:
+            site["quickLaunch"] = [{"title": n.get("Title"), "url": n.get("Url")} for n in self.sp.quick_launch()]
+        except SharePointError as e:
+            self._err("quick launch navigation", e)
+        self.list_data(site, sample_items)
+        _write(self.root / "site.json", site)
+        targets = pages or self.discover_pages(site)
+        fetched = [p for p in (self.page(t) for t in targets) if p]
+        self.assets(fetched)
+        self.manifest["summary"] = {"lists": len(self.manifest["lists"]), "pages": len(fetched),
+                                    "files": len(self.manifest["files"]), "errors": len(self.manifest["errors"])}
+        _write(self.root / "manifest.json", self.manifest)
+        return self.manifest

sp2react/spec/base.py

@@ -0,0 +1,80 @@
+"""Deterministic part of the spec: everything that can be derived without judgement.
+
+The spec agent receives this base and adds the rules/components that need interpretation
+(what a jQuery handler *means*). Keeping this part deterministic means it is never hallucinated.
+"""
+from __future__ import annotations
+
+import re
+
+from ..analysis.inventory import kebab
+from .model import Component, Endpoint, ListSpec, Rule, Screen, Spec
+
+MODES_ORDER = ["new", "edit", "display"]
+
+
+def _src(b: dict) -> list[str]:
+    return [f"{b['file']}:{b['line']}"] if b.get("line") else [b["file"]]
+
+
+def base_spec(inv: dict) -> Spec:
+    scr = inv["screen"]
+    primary = scr.get("list")
+    modes = [m for m in MODES_ORDER if any(p.get("mode") == m for p in inv["pages"])]
+    site_prefix = re.sub(r"^https?://[^/]+", "", inv["site"].get("url") or "").rstrip("/")
+    screen = Screen(
+        id=scr["id"], kind=scr["kind"], title=scr["title"], route=scr["route"], list=primary, modes=modes,
+        legacy_urls=[f"{site_prefix}/{p['path']}" for p in inv["pages"]],
+        stylesheets=inv.get("stylesheets", []),
+    )
+    rules: list[Rule] = []
+    by_kind: dict[str, list[dict]] = {}
+    for b in inv["behaviors"]:
+        by_kind.setdefault(b["kind"], []).append(b)
+
+    if scr["kind"] == "list-form":
+        form = Component(id="form", kind="form", title=scr["title"],
+                         props={"list": primary, "modes": modes, "toolbar": {"new": ["Save", "Cancel"], "edit": ["Save", "Cancel"], "display": ["Edit Item", "Close"]}},
+                         covers=[b["id"] for b in by_kind.get("attachments", [])])
+        screen.components.append(form)
+        # schema validations that translate mechanically
+        for b in by_kind.get("column_validation", []) + by_kind.get("list_validation", []):
+            ported = (b.get("detail") or {}).get("ported")
+            if not ported:
+                continue
+            n = len(rules) + 1
+            rules.append(Rule(id=f"S{n}", origin="schema", message=b.get("message"), presentation="inline",
+                              covers=[b["id"]], src=_src(b), list=primary, **ported))
+    else:
+        for i, frag in enumerate(inv.get("html_fragments", []), 1):
+            screen.components.append(Component(id=f"fragment{i}", kind="html", title=frag.get("web_part"), order=i,
+                                               props={"legacy_html": frag["html"][:4000], "page": frag["page"]}))
+        for b in by_kind.get("list_view", []):
+            d = b.get("detail") or {}
+            screen.components.append(Component(id=f"listview-{kebab(b['id'].split(':')[-2])}", kind="list_view",
+                                               title=b["code"].split("'")[1] if "'" in b["code"] else None,
+                                               region=d.get("zone"), props={"list": d.get("list"), "view_xml": d.get("view_xml")},
+                                               covers=[b["id"]], src=_src(b)))
+
+    # endpoints proposed from the data calls, merged by method+path
+    endpoints: dict[tuple, Endpoint] = {}
+    for c in inv["calls"]:
+        p = c.get("proposed") or {}
+        if not p.get("path") or p.get("method") not in ("GET", "POST", "PATCH", "DELETE"):
+            continue
+        key = (p["method"], p["path"])
+        if key not in endpoints:
+            m = re.match(r"/api/lists/(\w+)", p["path"])
+            eid = f"E{len(endpoints) + 1}"
+            endpoints[key] = Endpoint(id=eid, method=p["method"], path=p["path"], purpose=p.get("purpose", ""),
+                                      list=m.group(1) if m else None, body=p.get("body", []) or [])
+        e = endpoints[key]
+        e.covers.append(c["behavior"])
+        e.src.append(f"{c['behavior'].split('#L')[0]}:{c.get('line')}")
+        if p.get("query"):
+            e.query.setdefault("usages", []).append({k: v for k, v in p["query"].items() if v})
+    lists = {k: ListSpec(title=v["title"], primary=(v["role"] == "primary"), fields=v["fields"],
+                         attachments=v.get("attachments", False), views=v.get("views", {}))
+             for k, v in inv["lists"].items()}
+    roles = sorted({r["group"] for r in inv.get("roles", []) if r.get("referenced")})
+    return Spec(screen=screen, lists=lists, rules=rules, endpoints=list(endpoints.values()), roles=roles)