syntaxmatrix 2.6.4.4__py3-none-any.whl → 3.0.1__py3-none-any.whl

syntaxmatrix/client_docs.py

@@ -0,0 +1,237 @@
+# syntaxmatrix/client_docs.py
+from __future__ import annotations
+
+import os
+import re
+import html
+from dataclasses import dataclass
+from typing import List, Tuple
+
+from flask import abort, render_template
+from markupsafe import Markup
+
+
+@dataclass(frozen=True)
+class TocItem:
+    level: int
+    id: str
+    text: str
+
+
+_slug_rx = re.compile(r"[^a-z0-9\- ]+")
+_ws_rx = re.compile(r"\s+")
+
+
+def _slugify(text: str) -> str:
+    t = text.strip().lower()
+    t = _slug_rx.sub("", t)
+    t = _ws_rx.sub("-", t)
+    t = t.strip("-")
+    return t or "section"
+
+
+def _extract_headings(md: str) -> List[Tuple[int, str]]:
+    """
+    Extract ATX-style markdown headings (#, ##, ###, ...), ignoring fenced code blocks.
+    """
+    headings: List[Tuple[int, str]] = []
+    in_code = False
+    for line in md.splitlines():
+        if line.strip().startswith("```"):
+            in_code = not in_code
+            continue
+        if in_code:
+            continue
+
+        m = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
+        if not m:
+            continue
+        level = len(m.group(1))
+        title = m.group(2).strip()
+        # Avoid weird headings like "### ----"
+        if title and not all(ch in "-_=*" for ch in title):
+            headings.append((level, title))
+    return headings
+
+
+def _render_markdown_minimal(md: str, heading_ids: dict[str, str]) -> str:
+    """
+    Minimal markdown renderer (safe-by-default):
+    - headings, paragraphs, bullet lists, code fences, inline code, links, images
+    - everything else is HTML-escaped
+    """
+    lines = md.splitlines()
+    out: List[str] = []
+
+    in_code = False
+    code_lang = ""
+    code_buf: List[str] = []
+
+    in_ul = False
+
+    def flush_ul():
+        nonlocal in_ul
+        if in_ul:
+            out.append("</ul>")
+            in_ul = False
+
+    def flush_code():
+        nonlocal in_code, code_lang, code_buf
+        if not in_code:
+            return
+        code_text = "\n".join(code_buf)
+        out.append(
+            f'<pre class="smx-code"><code class="language-{html.escape(code_lang)}">'
+            f"{html.escape(code_text)}</code></pre>"
+        )
+        in_code = False
+        code_lang = ""
+        code_buf = []
+
+    def inline_fmt(s: str) -> str:
+        s = html.escape(s)
+
+        # inline code: `code`
+        s = re.sub(r"`([^`]+)`", lambda m: f"<code>{html.escape(m.group(1))}</code>", s)
+
+        # images: ![alt](url)
+        s = re.sub(
+            r"!\[([^\]]*)\]\(([^)]+)\)",
+            lambda m: f'<img alt="{html.escape(m.group(1))}" src="{html.escape(m.group(2))}" />',
+            s,
+        )
+
+        # links: [text](url)
+        s = re.sub(
+            r"\[([^\]]+)\]\(([^)]+)\)",
+            lambda m: f'<a href="{html.escape(m.group(2))}" target="_blank" rel="noopener noreferrer">{html.escape(m.group(1))}</a>',
+            s,
+        )
+
+        # bold **text**
+        s = re.sub(r"\*\*([^*]+)\*\*", r"<strong>\1</strong>", s)
+
+        # italics *text* (simple)
+        s = re.sub(r"(?<!\*)\*([^*]+)\*(?!\*)", r"<em>\1</em>", s)
+
+        return s
+
+    # Pre-map heading text to stable IDs (supports duplicates)
+    # heading_ids is already built with de-duplication.
+    for raw in lines:
+        line = raw.rstrip("\n")
+
+        # fenced code blocks
+        m_code = re.match(r"^\s*```(\w+)?\s*$", line)
+        if m_code:
+            flush_ul()
+            if in_code:
+                flush_code()
+            else:
+                in_code = True
+                code_lang = (m_code.group(1) or "").strip()
+                code_buf = []
+            continue
+
+        if in_code:
+            code_buf.append(line)
+            continue
+
+        # headings
+        m_h = re.match(r"^(#{1,6})\s+(.+?)\s*$", line)
+        if m_h:
+            flush_ul()
+            level = len(m_h.group(1))
+            text = m_h.group(2).strip()
+            hid = heading_ids.get(text) or _slugify(text)
+            out.append(
+                f'<h{level} id="{html.escape(hid)}" class="smx-h">{inline_fmt(text)}</h{level}>'
+            )
+            continue
+
+        # bullet list items (supports "-" or "*")
+        m_li = re.match(r"^\s*[-*]\s+(.+)\s*$", line)
+        if m_li:
+            if not in_ul:
+                out.append("<ul>")
+                in_ul = True
+            out.append(f"<li>{inline_fmt(m_li.group(1).strip())}</li>")
+            continue
+
+        # blank line ends lists/paragraph chunks
+        if line.strip() == "":
+            flush_ul()
+            out.append("")
+            continue
+
+        # blockquote (single-line)
+        m_bq = re.match(r"^\s*>\s+(.+)\s*$", line)
+        if m_bq:
+            flush_ul()
+            out.append(f"<blockquote>{inline_fmt(m_bq.group(1).strip())}</blockquote>")
+            continue
+
+        # horizontal rule
+        if re.match(r"^\s*---\s*$", line):
+            flush_ul()
+            out.append("<hr/>")
+            continue
+
+        # normal paragraph line
+        flush_ul()
+        out.append(f"<p>{inline_fmt(line.strip())}</p>")
+
+    flush_ul()
+    flush_code()
+
+    return "\n".join(out)
+
+
+def build_docs_html_and_toc(md: str) -> Tuple[str, List[TocItem]]:
+    headings = _extract_headings(md)
+
+    # Build stable unique IDs
+    used: dict[str, int] = {}
+    heading_ids: dict[str, str] = {}
+
+    toc: List[TocItem] = []
+    for level, title in headings:
+        base = _slugify(title)
+        n = used.get(base, 0)
+        used[base] = n + 1
+        hid = base if n == 0 else f"{base}-{n+1}"
+
+        # map by raw title (good enough for this README)
+        # if duplicates exist with same title, only first maps here; duplicates are still in TOC correctly
+        if title not in heading_ids:
+            heading_ids[title] = hid
+
+        toc.append(TocItem(level=level, id=hid, text=title))
+
+    html_content = _render_markdown_minimal(md, heading_ids)
+    return html_content, toc
+
+
+def register_client_docs_routes(app, client_dir: str) -> None:
+    """
+    Call this once during app initialisation.
+    Exposes:
+      GET /docs
+    """
+    @app.get("/docs")
+    def smx_client_docs():
+        readme_path = os.path.join(client_dir, "README.md")
+        if not os.path.exists(readme_path):
+            abort(404, description="README.md not found in client root")
+
+        with open(readme_path, "r", encoding="utf-8") as f:
+            md = f.read()
+
+        docs_html, toc = build_docs_html_and_toc(md)
+
+        return render_template(
+            "client_docs.html",
+            page_title="System Documentation",
+            toc=toc,
+            docs_html=Markup(docs_html),
+        )

syntaxmatrix/commentary.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-import os, io, re, json, base64
+import re, json
+import html as _html
+import re as _re
 from typing import Any, Dict, List, Optional
 
 from syntaxmatrix import profiles as _prof
@@ -114,11 +116,49 @@ def sniff_tables_from_html(html: str) -> List[Dict[str, Any]]:
     return tables
 
 
+def sniff_pre_text_from_html(html: str, *, max_lines: int = 18, max_chars: int = 900) -> List[str]:
+    """Extract short, useful plain-text snippets from <pre> blocks (metrics, tests, notes)."""
+    if not html:
+        return []
+    pres = _re.findall(r"<pre[^>]*>(.*?)</pre>", html, flags=_re.DOTALL | _re.IGNORECASE)
+    out: List[str] = []
+    for p in pres:
+        t = _strip_tags(p)
+        t = _html.unescape(t)
+        t = t.replace("\r\n", "\n").replace("\r", "\n")
+
+        for ln in t.split("\n"):
+            ln = (ln or "").strip()
+            if not ln:
+                continue
+            # keep only “result-ish” lines; drop giant dumps if any sneak in
+            if len(ln) > 260:
+                ln = ln[:260].rstrip() + "…"
+            out.append(ln)
+
+    # de-dup, preserve order
+    seen = set()
+    cleaned = []
+    for ln in out:
+        k = ln.lower()
+        if k in seen:
+            continue
+        seen.add(k)
+        cleaned.append(ln)
+
+    # cap total length
+    joined = "\n".join(cleaned)
+    joined = joined[:max_chars]
+    cleaned = joined.split("\n")[:max_lines]
+    return [c for c in (x.strip() for x in cleaned) if c]
+
+
 def build_display_summary(question: str,
                           mpl_axes: List[Dict[str, Any]],
                           html_blocks: List[str]) -> Dict[str, Any]:
     html_joined = "\n".join(str(b) for b in html_blocks)
     tables = sniff_tables_from_html(html_joined)
+    pre_snips = sniff_pre_text_from_html(html_joined)
 
     axes_clean=[]
     for ax in mpl_axes:
@@ -132,11 +172,14 @@ def build_display_summary(question: str,
     return {
         "question": (question or "").strip(),
         "axes": axes_clean,
-        "tables": tables
+        "tables": tables,
+        "text_snippets": pre_snips,
     }
 
 def _context_strings(context: Dict[str, Any]) -> List[str]:
     s = [context.get("question","")]
+    s += (context.get("text_snippets", []) or [])
+    
     for ax in context.get("axes", []) or []:
         s += [ax.get("title",""), ax.get("x_label",""), ax.get("y_label","")]
         s += (ax.get("legend", []) or [])
@@ -159,35 +202,63 @@ def phrase_commentary_vision(context: Dict[str, Any], images_b64: List[str]) ->
     send figures + text; otherwise fall back to a text-only prompt grounded by labels.
     """
     
-    _SYSTEM_VISION = ("""
-        You are a plots, graphs, and tables data analyst. You analyse and interprete in details and give your responses in plain english what the already-rendered plots and visuals mean as a response to the question. If the relevant information is made available, then, you must first answer the question explicitly and then proceed to explain the plots and tables. 
-        Use the information visible in the attached figures and the provided context strings (texts, tables, plot field names, labels). 
-        You should provide interpretations without prelude or preamble.
-    """)
+    _SYSTEM_VISION = """
+    You are an applied data analyst writing an answer to the user's question.
+
+    Your priority:
+    1) Answer the question directly (clear verdict first).
+    2) Justify the verdict using evidence from the figures and the provided context.
+    3) Keep it readable for a non-technical stakeholder.
+
+    Rules:
+    - Do NOT write a preamble.
+    - Do NOT narrate what a chart “looks like”; interpret it in relation to the question.
+    - Only use numbers if they are visible in the figures or included in the text snippets/context.
+    - Output must be safe HTML only: <b>, <p>, <ul>, <li>, <br>. No <style>, no <script>, no images.
+    """.strip()
 
     _USER_TMPL_VISION = """
-    question:
-    {q}
-
-    Visible context strings (tables, plots: titles, axes, legends, headers):
-    {ctx}
-
-    Write a comprehensive conclusion (~250-350 words) as follows:
-    - <b>Headline</b> 
-      2-3 sentence answering the question from an overview of all the output.
-    - <b>Evidence</b> 
-      8-10 bullets referencing the (output-texts/tables/panels/axes/legend groups) seen in the output. 
-      As you reference the visuals, you should interprete them in a way to show how they answer the question.
-    - <b>Limitations</b> 
-      1 bullet; avoid quoting numbers unless present in context.
-    - <b>Recommendations</b> 
-      1 bullet.
-    """
+    <b>Question</b>
+    <p>{q}</p>
+
+    <b>Available evidence</b>
+    <p><b>Text snippets:</b><br>{snips}</p>
+    <p><b>Plot/table context strings (titles, axes, legends, headers):</b><br>{ctx}</p>
+
+    Write the response in this exact structure:
+
+    <b>Answer</b>
+    <p>
+    Give a direct answer to the question in 2-4 sentences.
+    If the correct output is a decision (e.g., association vs none, higher vs lower, best model, significant vs not),
+    state it explicitly.
+    </p>
+    
+    <b>Key evidence</b>
+    <ul>
+    <li>5–8 bullets. Each bullet must link evidence → conclusion.</li>
+    <li>Reference plots/tables by their titles/axes/headers when possible.</li>
+    <li>Use numbers only if present in snippets or clearly visible.</li>
+    </ul>
+
+    <b>What this means</b>
+    <ul>
+    <li>2-4 bullets translating the finding into a practical takeaway.</li>
+    </ul>
+
+    <b>Limitations</b>
+    <ul><li>1-2 bullets (short).</li></ul>
+
+    <b>Next steps</b>
+    <ul><li>2 bullets (actionable).</li></ul>
+    """.strip()
 
     visible = _context_strings(context)
+    snips = "\n".join(context.get("text_snippets", []) or [])
     user = _USER_TMPL_VISION.format(
         q=context.get("question",""),
-        ctx=json.dumps(visible, ensure_ascii=False, indent=2)
+        snips=_html.escape(snips).replace("\n", "<br>"),
+        ctx=_html.escape(json.dumps(visible, ensure_ascii=False, indent=2)).replace("\n", "<br>")
     )
      
     commentary_profile = _prof.get_profile("imagetexter") or _prof.get_profile("admin")