sp2react 0.1.0__py3-none-any.whl

sp2react/checks.py

@@ -0,0 +1,105 @@
+"""Stage 8 (test against the original): build and test the generated app. Plain processes, no model."""
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class Check:
+    name: str
+    ok: bool
+    output: str
+    skipped: bool = False
+
+    def tail(self, n: int = 6000) -> str:
+        return self.output[-n:]
+
+
+def _run(cmd: list[str], cwd: Path, timeout: int, env: dict | None = None) -> tuple[int, str]:
+    try:
+        r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=timeout, env={**os.environ, **(env or {})})
+        return r.returncode, (r.stdout or "") + (r.stderr or "")
+    except subprocess.TimeoutExpired as e:
+        return 124, f"timed out after {timeout}s: {' '.join(cmd)}\n{(e.stdout or '')}"
+    except FileNotFoundError as e:
+        return 127, str(e)
+
+
+def backend_tests(out: Path, timeout: int = 900, env: dict | None = None) -> Check:
+    be = out / "backend"
+    db = be / "data" / "test.db"
+    if db.exists():
+        db.unlink()
+    code, txt = _run([sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", "tests"], be, timeout,
+                     {"PYTHONPATH": str(be), "DB_PATH": str(db), **(env or {})})
+    return Check("backend tests (generated API tests + rule vectors)", code == 0, txt)
+
+
+def frontend_install(out: Path, timeout: int = 1200) -> Check:
+    fe = out / "frontend"
+    if (fe / "node_modules" / ".bin" / "tsc").exists():
+        return Check("npm install", True, "node_modules present", skipped=True)
+    npm = shutil.which("npm")
+    if not npm:
+        return Check("npm install", False, "npm not found on PATH; install Node.js 20+ or set pipeline.run_builds: false", skipped=True)
+    code, txt = _run([npm, "install", "--no-audit", "--no-fund", "--loglevel=error"], fe, timeout)
+    return Check("npm install", code == 0, txt)
+
+
+def frontend_build(out: Path, timeout: int = 900) -> Check:
+    fe = out / "frontend"
+    npm = shutil.which("npm")
+    if not npm or not (fe / "node_modules").exists():
+        return Check("frontend build (tsc + vite)", False, "dependencies not installed", skipped=True)
+    code, txt = _run([npm, "run", "build", "--silent"], fe, timeout)
+    return Check("frontend build (tsc + vite)", code == 0, txt)
+
+
+def frontend_rule_vectors(out: Path, timeout: int = 120) -> Check:
+    node = shutil.which("node")
+    fe = out / "frontend"
+    if not node:
+        return Check("rule vectors (TypeScript engine)", False, "node not found", skipped=True)
+    code, txt = _run([node, "scripts/check-rules.ts"], fe, timeout)
+    if code != 0 and ("Unknown file extension" in txt or "ERR_UNKNOWN_FILE_EXTENSION" in txt):
+        code, txt = _run([node, "--experimental-strip-types", "scripts/check-rules.ts"], fe, timeout)
+        if code != 0 and "bad option" in txt:
+            return Check("rule vectors (TypeScript engine)", False, "Node.js >= 22.6 is needed to run the TypeScript vectors directly", skipped=True)
+    return Check("rule vectors (TypeScript engine)", code == 0, txt)
+
+
+def run_all(out: Path, run_builds: bool = True, env: dict | None = None) -> list[Check]:
+    checks = [backend_tests(out, env=env)]
+    if run_builds:
+        inst = frontend_install(out)
+        checks.append(inst)
+        if inst.ok:
+            checks.append(frontend_build(out))
+        checks.append(frontend_rule_vectors(out))
+    return checks
+
+
+def failures_by_screen(checks: list[Check], screen_ids: list[str], handlers: dict[str, str]) -> dict[str, str]:
+    """Route failure output to the screen whose files are named in it; '' key = infrastructure."""
+    out: dict[str, list[str]] = {}
+    for c in checks:
+        if c.ok or c.skipped:
+            continue
+        hit = False
+        for sid in screen_ids:
+            if re.search(rf"screens/{re.escape(sid)}/", c.output) or re.search(rf"test_{re.escape(sid.lower().replace('-', '_'))}", c.output):
+                out.setdefault(sid, []).append(f"## {c.name}\n{c.tail()}")
+                hit = True
+        for handler, sid in handlers.items():
+            if re.search(rf"custom/{re.escape(handler)}\.", c.output):
+                out.setdefault(sid, []).append(f"## {c.name}\n{c.tail()}")
+                hit = True
+        if not hit:
+            out.setdefault("", []).append(f"## {c.name}\n{c.tail()}")
+    return {k: "\n\n".join(v) for k, v in out.items()}

sp2react/cli.py

@@ -0,0 +1,188 @@
+"""sp2react command line."""
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+
+from . import __version__
+from .config import TEMPLATE, ConfigError, load_config
+
+DEFAULT_CONFIG = "sp2react.yaml"
+
+
+def _pipeline(args):
+    from .pipeline import Pipeline
+    cfg = load_config(args.config)
+    if getattr(args, "auto_approve", False):
+        cfg.pipeline.auto_approve = True
+    if getattr(args, "no_builds", False):
+        cfg.pipeline.run_builds = False
+    return Pipeline(cfg, progress=lambda m: print(m, flush=True))
+
+
+def cmd_init(args) -> int:
+    p = Path(args.config)
+    if p.exists() and not args.force:
+        print(f"{p} already exists (use --force to overwrite)")
+        return 1
+    p.write_text(TEMPLATE, encoding="utf-8")
+    print(f"Wrote {p}. Fill in sharepoint.site_url and the llm section, set the secret environment variables, then run: sp2react doctor")
+    return 0
+
+
+def cmd_doctor(args) -> int:
+    from .llm.providers import make_llm
+    from .sharepoint.client import SharePointClient
+    cfg = load_config(args.config)
+    ok = True
+    try:
+        web = SharePointClient(cfg.sharepoint).web()
+        print(f"SharePoint: OK - '{web.get('Title')}' ({web.get('ServerRelativeUrl')})")
+    except Exception as e:
+        ok = False
+        print(f"SharePoint: FAILED - {e}")
+    try:
+        llm = make_llm(cfg.llm)
+        try:
+            models = llm.models()
+            claude = [m for m in models if "claude" in (m or "").lower()]
+            print(f"Model provider ({cfg.llm.provider}): OK - {len(models)} model(s){'; Claude: ' + ', '.join(claude) if claude else ''}")
+            if cfg.llm.model and models and cfg.llm.model not in models:
+                print(f"  warning: configured model '{cfg.llm.model}' is not in the provider's list")
+        except Exception as e:
+            print(f"Model provider ({cfg.llm.provider}): could not list models ({e}); trying a one-line completion")
+        c = llm.complete("Reply with the single word: ready", [{"role": "user", "content": "ping"}], max_tokens=10)
+        print(f"Model call: OK - {cfg.llm.model or cfg.llm.deployment} replied '{c.text.strip()[:40]}'")
+    except Exception as e:
+        ok = False
+        print(f"Model provider: FAILED - {e}")
+    return 0 if ok else 2
+
+
+def cmd_run(args) -> int:
+    p = _pipeline(args)
+    p.run(pages=args.page or None, offline=args.offline, until=args.until)
+    print(f"\nReport: {p.ws / 'report.md'}\nApp:    {p.out}")
+    return 0
+
+
+def cmd_stage(args) -> int:
+    from .pipeline import write_report
+    p = _pipeline(args)
+    scrs = p.screens(args.page or None)
+    sids = [s["id"] for s in scrs] if not args.screen else args.screen
+    if args.stage == "extract":
+        p.stage_inputs(False)
+        p.stage_extract(args.page or None, False)
+    elif args.stage == "inventory":
+        p.stage_inventory(args.page or None)
+    elif args.stage == "spec":
+        p.stage_spec(sids)
+    elif args.stage == "convert":
+        p.stage_convert(sids)
+    elif args.stage == "review":
+        p.stage_review(sids)
+    elif args.stage == "test":
+        p.stage_test_and_fix(sids)
+    elif args.stage == "migrate":
+        specs = [p.load_spec(s) for s in p.approved_sids()]
+        p._offline = False
+        from .scaffold.generate import generate
+        generate(p.out, specs, p.legacy, p.stage_migrate(specs))
+    elif args.stage == "map-db":
+        specs = [p.load_spec(s) for s in p.approved_sids()] or [p.load_spec(sc["id"]) for sc in p.screens() if (p.ws / "specs" / f"{sc['id']}.json").exists()]
+        ok = p.stage_map_db(specs)
+        print(f"{'Mapping ready' if ok else 'Mapping needs attention'}: {p.cfg.database.mapping} and {p.ws / 'db-mapping.md'}")
+    elif args.stage == "reconcile":
+        rep = p.stage_reconcile([p.load_spec(s) for s in p.approved_sids()])
+        print(f"{'All lists match' if rep['ok'] else 'Mismatches found'} - see {p.ws / 'reconciliation.md'}")
+    write_report(p)
+    return 0
+
+
+def cmd_approve(args) -> int:
+    from .pipeline import write_report
+    p = _pipeline(args)
+    gate = "screen" if args.screen_gate else "spec"
+    if gate == "spec" and p.status(args.screen_id, "spec") != "done":
+        print(f"The spec for {args.screen_id} is not done yet")
+        return 1
+    if gate == "screen" and p.status(args.screen_id, "test") != "done":
+        print(f"{args.screen_id} has not passed testing yet")
+        return 1
+    p.approve(args.screen_id, gate, args.by or "")
+    write_report(p)
+    return 0
+
+
+def cmd_status(args) -> int:
+    from .pipeline import STAGES
+    p = _pipeline(args)
+    if args.json:
+        print(json.dumps(p.state, indent=2))
+        return 0
+    for k in ("inputs", "extract"):
+        s = p.state["global"].get(k)
+        if s:
+            print(f"{k:<10} {s['status']:<9} {s.get('detail', '')}")
+    for sid, st in p.state["screens"].items():
+        print(f"\n{sid}")
+        for s in STAGES[2:]:
+            if st.get(s):
+                print(f"  {s:<13} {st[s]['status']:<9} {st[s].get('detail', '')[:110]}")
+    return 0
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="sp2react", description="Convert classic SharePoint ASPX/jQuery forms and pages to React + FastAPI with AI agents.")
+    ap.add_argument("--version", action="version", version=__version__)
+    ap.add_argument("-c", "--config", default=DEFAULT_CONFIG)
+    ap.add_argument("-v", "--verbose", action="store_true")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("init", help="write a starter sp2react.yaml")
+    s.add_argument("--force", action="store_true")
+    s.set_defaults(fn=cmd_init)
+
+    s = sub.add_parser("doctor", help="check the SharePoint connection and the model provider (lists Bridger's Claude models)")
+    s.set_defaults(fn=cmd_doctor)
+
+    s = sub.add_parser("run", help="run every stage up to the next approval gate")
+    s.add_argument("--page", action="append", help="site-relative or server-relative page URL (repeatable); default: all list forms + SitePages")
+    s.add_argument("--offline", action="store_true", help="reuse the extraction already in the workspace")
+    s.add_argument("--until", default="approve", choices=["inputs", "extract", "inventory", "spec", "approve_spec", "convert", "review", "test", "fix", "approve"])
+    s.add_argument("--auto-approve", action="store_true", help="skip the human gates (CI / demos)")
+    s.add_argument("--no-builds", action="store_true", help="skip npm install/build")
+    s.set_defaults(fn=cmd_run)
+
+    for name in ("extract", "inventory", "spec", "map-db", "convert", "review", "test", "migrate", "reconcile"):
+        s = sub.add_parser(name, help=f"run only the {name} stage")
+        s.add_argument("--page", action="append")
+        s.add_argument("--screen", action="append")
+        s.add_argument("--no-builds", action="store_true")
+        s.set_defaults(fn=cmd_stage, stage=name)
+
+    s = sub.add_parser("approve", help="approve a screen's spec (default) or the finished screen (--screen)")
+    s.add_argument("screen_id")
+    s.add_argument("--screen", dest="screen_gate", action="store_true")
+    s.add_argument("--by", help="name to record with the approval")
+    s.set_defaults(fn=cmd_approve)
+
+    s = sub.add_parser("status", help="show every screen's stage status")
+    s.add_argument("--json", action="store_true")
+    s.set_defaults(fn=cmd_status)
+
+    args = ap.parse_args(argv)
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.WARNING, format="%(levelname)s %(name)s: %(message)s")
+    try:
+        return args.fn(args)
+    except ConfigError as e:
+        print(f"Configuration problem: {e}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

sp2react/config.py

@@ -0,0 +1,172 @@
+"""Configuration: one YAML file, secrets only through environment variables."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Literal, Optional, Union
+
+import yaml
+from pydantic import BaseModel, Field, field_validator
+
+
+class ConfigError(RuntimeError):
+    pass
+
+
+class SharePointConfig(BaseModel):
+    site_url: str = Field(..., description="Full URL of the site, e.g. https://sp.example.com/sites/SEA")
+    auth: Literal["ntlm", "cookie", "bearer", "none"] = "ntlm"
+    username_env: str = "SP_USERNAME"          # DOMAIN\\user for NTLM
+    password_env: str = "SP_PASSWORD"
+    cookie_env: str = "SP_COOKIE"              # e.g. "FedAuth=...; rtFa=..." copied from a signed-in browser
+    token_env: str = "SP_TOKEN"                # bearer token (SharePoint Online / Entra)
+    verify_ssl: Union[bool, str] = True        # True, False, or path to a corporate CA bundle
+    timeout: int = 60
+    sample_items: int = 200                    # rows exported per list (lookup values, seed data)
+    extra_headers: dict[str, str] = Field(default_factory=dict)
+
+    @field_validator("site_url")
+    @classmethod
+    def _strip(cls, v: str) -> str:
+        return v.rstrip("/")
+
+
+class LLMConfig(BaseModel):
+    provider: Literal["bridger", "openai", "azure_openai", "anthropic", "scripted"] = "bridger"
+    api_format: Literal["anthropic", "openai"] = "anthropic"  # bridger only: which wire format Bridger serves Claude on
+    base_url: Optional[str] = None             # bridger: http://host:port ; azure: https://<res>.openai.azure.com
+    model: Optional[str] = None                # model id as the provider lists it (run `sp2react doctor`); azure uses `deployment`
+    api_key_env: Optional[str] = None          # env var holding the key/token; omit if Bridger needs none
+    deployment: Optional[str] = None           # azure_openai only
+    api_version: str = "2024-10-21"            # azure_openai only
+    temperature: float = 0.0
+    max_tokens: int = 16000
+    timeout: int = 600                         # seconds per model call
+    json_mode: bool = True                     # send response_format=json_object where supported
+    extra_headers: dict[str, str] = Field(default_factory=dict)
+    verify_ssl: Union[bool, str] = True
+
+
+class PipelineConfig(BaseModel):
+    workspace: str = "./sp2react-work"         # extraction, inventory, spec, agent transcripts, state
+    output: str = "./converted-app"            # generated React + FastAPI app
+    max_concurrency: int = 4                   # parallel agent calls
+    agent_retries: int = 2                     # re-asks per agent task when its output fails validation
+    agent_timeout: int = 900                   # hard limit per agent task (seconds)
+    max_fix_iterations: int = 5
+    auto_approve: bool = False                 # skip the human gates (spec + screen approval)
+    run_builds: bool = True                    # npm install/build + pytest on the generated app
+    data_source: Literal["database", "sqlite", "sharepoint"] = "database"
+    # database   = your existing database (MySQL / SQL Server / PostgreSQL) through a list->table mapping;
+    #              no list items are ever read from SharePoint
+    # sqlite     = sp2react migrates the list items into a SQLite copy (needs sharepoint item reads)
+    # sharepoint = the generated API reads and writes the live lists
+
+
+class DatabaseConfig(BaseModel):
+    url_env: str = "SP2REACT_DB_URL"           # env var with a SQLAlchemy URL, e.g. mysql+pymysql://user:pass@host:3306/sea
+    schema_name: Optional[str] = None          # database schema, if not the default
+    mapping: str = "./db-mapping.json"         # list -> table / field -> column mapping (sp2react map-db drafts it)
+
+
+class SecurityConfig(BaseModel):
+    allowed_hosts: list[str] = Field(default_factory=list)   # extra hosts beyond the SharePoint site and model endpoint
+    sharepoint_item_reads: Optional[bool] = None             # None: allowed only when data_source is sqlite
+
+
+class Config(BaseModel):
+    sharepoint: SharePointConfig
+    llm: LLMConfig = Field(default_factory=LLMConfig)
+    pipeline: PipelineConfig = Field(default_factory=PipelineConfig)
+    database: DatabaseConfig = Field(default_factory=DatabaseConfig)
+    security: SecurityConfig = Field(default_factory=SecurityConfig)
+
+    @property
+    def item_reads_allowed(self) -> bool:
+        """Whether sp2react may read list ITEMS (business data) from SharePoint over HTTP."""
+        if self.security.sharepoint_item_reads is not None:
+            return self.security.sharepoint_item_reads
+        return self.pipeline.data_source == "sqlite"
+
+    @property
+    def workspace(self) -> Path:
+        return Path(self.pipeline.workspace).resolve()
+
+    @property
+    def output(self) -> Path:
+        return Path(self.pipeline.output).resolve()
+
+
+def secret(env_name: Optional[str], what: str, required: bool = True) -> Optional[str]:
+    if not env_name:
+        if required:
+            raise ConfigError(f"No environment variable configured for {what}.")
+        return None
+    value = os.environ.get(env_name)
+    if not value and required:
+        raise ConfigError(f"Set the environment variable {env_name} ({what}).")
+    return value
+
+
+def load_config(path: Union[str, Path]) -> Config:
+    p = Path(path)
+    if not p.exists():
+        raise ConfigError(f"Config file not found: {p}. Run `sp2react init` to create one.")
+    data = yaml.safe_load(p.read_text(encoding="utf-8")) or {}
+    try:
+        cfg = Config.model_validate(data)
+    except Exception as e:  # pydantic ValidationError -> readable message
+        raise ConfigError(f"Invalid config {p}:\n{e}") from e
+    # Relative workspace/output paths are relative to the config file, not the cwd
+    base = p.resolve().parent
+    for attr in ("workspace", "output"):
+        val = getattr(cfg.pipeline, attr)
+        if not Path(val).is_absolute():
+            setattr(cfg.pipeline, attr, str((base / val).resolve()))
+    if not Path(cfg.database.mapping).is_absolute():
+        cfg.database.mapping = str((base / cfg.database.mapping).resolve())
+    return cfg
+
+
+TEMPLATE = """\
+# sp2react configuration. Secrets are read from environment variables, never stored here.
+sharepoint:
+  site_url: https://sharepoint.example.com/sites/SEA
+  auth: ntlm                 # ntlm | cookie | bearer | none
+  username_env: SP_USERNAME  # DOMAIN\\\\user  (ntlm)
+  password_env: SP_PASSWORD  #               (ntlm)
+  cookie_env: SP_COOKIE      # "FedAuth=...; rtFa=..." from a signed-in browser (cookie)
+  token_env: SP_TOKEN        # bearer token (bearer)
+  verify_ssl: true           # or a path to your corporate CA bundle
+  sample_items: 200
+
+llm:
+  provider: bridger          # bridger | anthropic | openai | azure_openai
+  api_format: anthropic      # how Bridger serves Claude: anthropic (/v1/messages) or openai (/v1/chat/completions)
+  base_url: http://localhost:4141
+  model: claude-sonnet-4     # exact Claude model id as Bridger lists it: run `sp2react doctor`
+  api_key_env: BRIDGER_TOKEN # omit if Bridger needs no token
+  # azure_openai: base_url: https://<resource>.openai.azure.com, deployment: <name>, api_key_env: AZURE_OPENAI_API_KEY
+  # anthropic:    model: claude-sonnet-5-5, api_key_env: ANTHROPIC_API_KEY
+  # openai:       base_url: https://api.openai.com/v1, model: <model>, api_key_env: OPENAI_API_KEY
+
+pipeline:
+  workspace: ./sp2react-work
+  output: ./converted-app
+  max_concurrency: 4
+  agent_retries: 2
+  agent_timeout: 900
+  max_fix_iterations: 5
+  auto_approve: false
+  run_builds: true
+  data_source: database      # database: your existing MySQL/SQL Server tables (no list items read over HTTP)
+                             # sqlite: sp2react migrates the items itself | sharepoint: API uses the live lists
+
+database:
+  url_env: SP2REACT_DB_URL   # e.g. mysql+pymysql://user:password@db-host:3306/sea  (read from the environment)
+  mapping: ./db-mapping.json # drafted by `sp2react map-db`, reviewed by you
+
+security:
+  allowed_hosts: []          # every request is refused unless it goes to the SharePoint site or the model endpoint
+  # sharepoint_item_reads: false   # default: false unless data_source is sqlite
+"""