syntaxmatrix 2.6.4.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

syntaxmatrix/commentary.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-import os, io, re, json, base64
+import re, json
+import html as _html
+import re as _re
 from typing import Any, Dict, List, Optional
 
 from syntaxmatrix import profiles as _prof
@@ -114,11 +116,49 @@ def sniff_tables_from_html(html: str) -> List[Dict[str, Any]]:
     return tables
 
 
+def sniff_pre_text_from_html(html: str, *, max_lines: int = 18, max_chars: int = 900) -> List[str]:
+    """Extract short, useful plain-text snippets from <pre> blocks (metrics, tests, notes)."""
+    if not html:
+        return []
+    pres = _re.findall(r"<pre[^>]*>(.*?)</pre>", html, flags=_re.DOTALL | _re.IGNORECASE)
+    out: List[str] = []
+    for p in pres:
+        t = _strip_tags(p)
+        t = _html.unescape(t)
+        t = t.replace("\r\n", "\n").replace("\r", "\n")
+
+        for ln in t.split("\n"):
+            ln = (ln or "").strip()
+            if not ln:
+                continue
+            # keep only “result-ish” lines; drop giant dumps if any sneak in
+            if len(ln) > 260:
+                ln = ln[:260].rstrip() + "…"
+            out.append(ln)
+
+    # de-dup, preserve order
+    seen = set()
+    cleaned = []
+    for ln in out:
+        k = ln.lower()
+        if k in seen:
+            continue
+        seen.add(k)
+        cleaned.append(ln)
+
+    # cap total length
+    joined = "\n".join(cleaned)
+    joined = joined[:max_chars]
+    cleaned = joined.split("\n")[:max_lines]
+    return [c for c in (x.strip() for x in cleaned) if c]
+
+
 def build_display_summary(question: str,
                           mpl_axes: List[Dict[str, Any]],
                           html_blocks: List[str]) -> Dict[str, Any]:
     html_joined = "\n".join(str(b) for b in html_blocks)
     tables = sniff_tables_from_html(html_joined)
+    pre_snips = sniff_pre_text_from_html(html_joined)
 
     axes_clean=[]
     for ax in mpl_axes:
@@ -132,11 +172,14 @@ def build_display_summary(question: str,
     return {
         "question": (question or "").strip(),
         "axes": axes_clean,
-        "tables": tables
+        "tables": tables,
+        "text_snippets": pre_snips,
     }
 
 def _context_strings(context: Dict[str, Any]) -> List[str]:
     s = [context.get("question","")]
+    s += (context.get("text_snippets", []) or [])
+    
     for ax in context.get("axes", []) or []:
         s += [ax.get("title",""), ax.get("x_label",""), ax.get("y_label","")]
         s += (ax.get("legend", []) or [])
@@ -159,35 +202,63 @@ def phrase_commentary_vision(context: Dict[str, Any], images_b64: List[str]) ->
     send figures + text; otherwise fall back to a text-only prompt grounded by labels.
     """
     
-    _SYSTEM_VISION = ("""
-        You are a plots, graphs, and tables data analyst. You analyse and interprete in details and give your responses in plain english what the already-rendered plots and visuals mean as a response to the question. If the relevant information is made available, then, you must first answer the question explicitly and then proceed to explain the plots and tables. 
-        Use the information visible in the attached figures and the provided context strings (texts, tables, plot field names, labels). 
-        You should provide interpretations without prelude or preamble.
-    """)
+    _SYSTEM_VISION = """
+    You are an applied data analyst writing an answer to the user's question.
+
+    Your priority:
+    1) Answer the question directly (clear verdict first).
+    2) Justify the verdict using evidence from the figures and the provided context.
+    3) Keep it readable for a non-technical stakeholder.
+
+    Rules:
+    - Do NOT write a preamble.
+    - Do NOT narrate what a chart “looks like”; interpret it in relation to the question.
+    - Only use numbers if they are visible in the figures or included in the text snippets/context.
+    - Output must be safe HTML only: <b>, <p>, <ul>, <li>, <br>. No <style>, no <script>, no images.
+    """.strip()
 
     _USER_TMPL_VISION = """
-    question:
-    {q}
-
-    Visible context strings (tables, plots: titles, axes, legends, headers):
-    {ctx}
-
-    Write a comprehensive conclusion (~250-350 words) as follows:
-    - <b>Headline</b> 
-      2-3 sentence answering the question from an overview of all the output.
-    - <b>Evidence</b> 
-      8-10 bullets referencing the (output-texts/tables/panels/axes/legend groups) seen in the output. 
-      As you reference the visuals, you should interprete them in a way to show how they answer the question.
-    - <b>Limitations</b> 
-      1 bullet; avoid quoting numbers unless present in context.
-    - <b>Recommendations</b> 
-      1 bullet.
-    """
+    <b>Question</b>
+    <p>{q}</p>
+
+    <b>Available evidence</b>
+    <p><b>Text snippets:</b><br>{snips}</p>
+    <p><b>Plot/table context strings (titles, axes, legends, headers):</b><br>{ctx}</p>
+
+    Write the response in this exact structure:
+
+    <b>Answer</b>
+    <p>
+    Give a direct answer to the question in 2-4 sentences.
+    If the correct output is a decision (e.g., association vs none, higher vs lower, best model, significant vs not),
+    state it explicitly.
+    </p>
+    
+    <b>Key evidence</b>
+    <ul>
+    <li>5–8 bullets. Each bullet must link evidence → conclusion.</li>
+    <li>Reference plots/tables by their titles/axes/headers when possible.</li>
+    <li>Use numbers only if present in snippets or clearly visible.</li>
+    </ul>
+
+    <b>What this means</b>
+    <ul>
+    <li>2-4 bullets translating the finding into a practical takeaway.</li>
+    </ul>
+
+    <b>Limitations</b>
+    <ul><li>1-2 bullets (short).</li></ul>
+
+    <b>Next steps</b>
+    <ul><li>2 bullets (actionable).</li></ul>
+    """.strip()
 
     visible = _context_strings(context)
+    snips = "\n".join(context.get("text_snippets", []) or [])
     user = _USER_TMPL_VISION.format(
         q=context.get("question",""),
-        ctx=json.dumps(visible, ensure_ascii=False, indent=2)
+        snips=_html.escape(snips).replace("\n", "<br>"),
+        ctx=_html.escape(json.dumps(visible, ensure_ascii=False, indent=2)).replace("\n", "<br>")
     )
      
     commentary_profile = _prof.get_profile("imagetexter") or _prof.get_profile("admin")

syntaxmatrix/core.py

@@ -16,7 +16,7 @@ from .file_processor import process_admin_pdf_files
 from google.genai import types
 from .vector_db import query_embeddings
 from .vectorizer import embed_text
-from typing import List, Generator
+from typing import List, Generator, Optional
 from .auth import init_auth_db
 from . import profiles as _prof
 from syntaxmatrix.smiv import SMIV
@@ -27,17 +27,16 @@ from html import unescape
 from .plottings import render_plotly, pyplot, describe_plotly, describe_matplotlib
 from threading import RLock
 from syntaxmatrix.settings.model_map import GPT_MODELS_LATEST
+from syntaxmatrix.settings.client_items import read_client_file, getenv_api_key
+from syntaxmatrix.plugins.plugin_manager import PluginManager
+from .premium import FeatureGate, ensure_premium_state
+from pathlib import Path
 from syntaxmatrix.settings.prompts import(
     SMXAI_CHAT_IDENTITY, 
     SMXAI_CHAT_INSTRUCTIONS, 
     SMXAI_WEBSITE_DESCRIPTION,
 )
-from syntaxmatrix.settings.client_items import read_client_file, getenv_api_key
-
-from .premium import FeatureGate
-from .plugin_manager import PluginManager
-
-
+ 
 # ──────── framework‐local storage paths ────────
 # this ensures the key & data always live under the package dir,
 _CLIENT_DIR = detect_project_root()
@@ -59,7 +58,7 @@ EDA_OUTPUT = {}  # global buffer for EDA output by session
 
 class SyntaxMUI:
     def __init__(self, 
-           host="127.0.0.1", 
+            host="127.0.0.1", 
             port="5080", 
             user_icon="👩🏿‍🦲",
             bot_icon="<img src='/static/icons/bot_icon.png' width=20' alt='bot'/>",
@@ -67,10 +66,12 @@ class SyntaxMUI:
             site_logo="<img src='/static/icons/logo.png' width='45' alt='logo'/>",           
             site_title="SyntaxMatrix", 
             project_name="smxAI", 
-            theme_name="light",
+            theme_name="chark",
             ui_mode = "default"
         ):
-        self.app = Flask(__name__)         
+        self.app = Flask(__name__)     
+        self._client_dir = Path(_CLIENT_DIR).resolve()
+        self._client_root = self._client_dir.parent    
         self.host = host
         self.port = port
 
@@ -82,7 +83,6 @@ class SyntaxMUI:
         self.bot_icon = bot_icon
         self.site_title = site_title
         self.project_name = project_name
-
         self._default_site_logo = self.site_logo
         self._default_favicon = self.favicon
         self._default_bot_icon = self.bot_icon
@@ -93,25 +93,45 @@ class SyntaxMUI:
         self.theme_toggle_enabled = False
         self.user_files_enabled = False
         self.registration_enabled = False
+        self.site_documentation_enabled = False
+        self.ml_lab_enabled = False
+
         self.smxai_identity = SMXAI_CHAT_IDENTITY
         self.smxai_instructions = SMXAI_CHAT_INSTRUCTIONS    
         self.website_description = SMXAI_WEBSITE_DESCRIPTION
+        # Preserve framework defaults so client branding can safely override and reset.
+        self._default_smxai_identity = self.smxai_identity
+        self._default_smxai_instructions = self.smxai_instructions
+        self._default_website_description = self.website_description
+
         self._eda_output = {}      # {chat_id: html}
         self._eda_lock = RLock()
         
         db.init_db()
+        self.db = db
         self.page = ""
         self.pages = db.get_pages()
+        
         init_auth_db() 
+        try:
+            ensure_premium_state(db=db, client_dir=str(_CLIENT_DIR), trial_days=7)
+        except Exception:
+            pass
 
         self.widgets = OrderedDict()
-        self.theme = DEFAULT_THEMES.get(theme_name, DEFAULT_THEMES["light"])     
-        self.system_output_buffer = ""  # Ephemeral buffer initialized  
+        self.theme = DEFAULT_THEMES.get(theme_name, DEFAULT_THEMES["chark"])     
+        self.system_output_buffer = ""  # Ephemeral buffer initialised  
         self.app_token = str(uuid.uuid4())  # NEW: Unique token for each app launch.
         self.admin_pdf_chunks = {}   # In-memory store for admin PDF chunks
-        self.user_file_chunks = {}  # In-memory store of user‑uploaded chunks, scoped per chat session
-        
+        self.user_file_chunks = {}  # In-memory store of user‑uploaded chunks, scoped per chat 
         self._last_llm_usage = None 
+
+        # Apply persisted feature flags + theme (fail-soft)
+        try:
+            self._apply_feature_flags_from_db()
+        except Exception:
+            pass
+
         routes.setup_routes(self)
 
         # Apply client branding overrides if present on disk
@@ -182,7 +202,7 @@ class SyntaxMUI:
         if not hasattr(self, "_recent_visual_summaries"):
             self._recent_visual_summaries = []
         # keep last 6
-        self._recent_visual_summaries = (self._recent_visual_summaries + [summary])[-6:]
+        self._recent_visual_summaries = (self._recent_visual_summaries + [summary]) # [-6:]
 
     def set_plottings(self, fig_or_html, note=None):
         # prefer current chat id; fall back to per-browser sid; finally "default"
@@ -340,18 +360,29 @@ class SyntaxMUI:
             DEFAULT_THEMES[theme_name] = theme
             self.theme = DEFAULT_THEMES[theme_name]
         else:
-            self.theme = DEFAULT_THEMES["light"]
-            self.error("Theme must be 'light', 'dark', or a custom dict.")
+            self.theme = DEFAULT_THEMES["chark"]
+            self.error("Theme must be 'chark', 'light', 'dark', or a custom dict.")
 
-    
-    def enable_theme_toggle(self):
-        self.theme_toggle_enabled = True 
-    
-    def enable_user_files(self):
-        self.user_files_enabled = True
+    def enable_theme_toggle(self, bul: bool = True):
+        self.theme_toggle_enabled = bool(bul)
+        return self.theme_toggle_enabled
+
+    def enable_user_files(self, bul: bool = True):
+        self.user_files_enabled = bool(bul)
+        return self.user_files_enabled
+
+    def enable_registration(self, bul: bool = True):
+        self.registration_enabled = bool(bul)
+        return self.registration_enabled
+
+    def enable_site_documentation(self, bul: bool = True):
+        self.site_documentation_enabled = bool(bul)
+        return self.site_documentation_enabled
+
+    def enable_ml_lab(self, bul: bool = True):
+        self.ml_lab_enabled = bool(bul)
+        return self.ml_lab_enabled
 
-    def enable_registration(self):
-        self.registration_enabled = True
 
     def _apply_feature_flags_from_db(self):
         """
@@ -364,8 +395,25 @@ class SyntaxMUI:
             stream_v = db.get_setting("feature.stream_mode", "1")
             user_files_v = db.get_setting("feature.user_files", "1")
 
+            # NEW defaults are all False (0)
+            docs_v = db.get_setting("feature.site_documentation", "0")
+            ml_v = db.get_setting("feature.ml_lab", "0")
+            reg_v = db.get_setting("feature.registration", "0")
+            theme_toggle_v = db.get_setting("feature.theme_toggle", "0")
+
+            # Theme is a choice (default light)
+            theme_name = db.get_setting("branding.theme_name", "light")
+
             self.is_streaming = _truthy(stream_v)
             self.user_files_enabled = _truthy(user_files_v)
+
+            self.site_documentation_enabled = _truthy(docs_v)
+            self.ml_lab_enabled = _truthy(ml_v)
+            self.registration_enabled = _truthy(reg_v)
+            self.theme_toggle_enabled = _truthy(theme_toggle_v)
+
+            # Apply the chosen theme to the instance
+            self.set_theme(str(theme_name or "light").strip().lower() or "light")
         except Exception:
             # Keep defaults if DB isn't ready for any reason
             pass
@@ -399,6 +447,31 @@ class SyntaxMUI:
         use it; otherwise keep the framework defaults.
         Also pulls site_title and project_name from app_settings.
         """
+
+        # Premium gating: ignore custom branding unless entitled.
+        try:
+            if hasattr(self, "feature_gate") and self.feature_gate and (not self.feature_gate.enabled("branding_controls")):
+                self.site_logo = getattr(self, "_default_site_logo", self.site_logo)
+                self.favicon = getattr(self, "_default_favicon", self.favicon)
+                self.bot_icon = getattr(self, "_default_bot_icon", self.bot_icon)
+                try:
+                    self.set_smxai_identity("")
+                    self.set_smxai_instructions("")
+                    self.set_website_description(getattr(self, "_default_website_description", self.website_description))
+                except Exception:
+                    pass
+                self.site_title = getattr(self, "_default_site_title", self.site_title)
+                self.project_name = getattr(self, "_default_project_name", self.project_name)
+                return
+        except Exception:
+            # Fail closed: if we cannot resolve entitlements, do not apply custom branding.
+            self.site_logo = getattr(self, "_default_site_logo", self.site_logo)
+            self.favicon = getattr(self, "_default_favicon", self.favicon)
+            self.bot_icon = getattr(self, "_default_bot_icon", self.bot_icon)
+            self.site_title = getattr(self, "_default_site_title", self.site_title)
+            self.project_name = getattr(self, "_default_project_name", self.project_name)
+            return
+
         branding_dir = os.path.join(_CLIENT_DIR, "branding")
 
         def _pick_any(*basenames: str):
@@ -432,6 +505,18 @@ class SyntaxMUI:
             self.bot_icon = f"<img src='/branding/{bot_fn}' width='20' alt='bot'/>"
         else:
             self.bot_icon = getattr(self, "_default_bot_icon", self.bot_icon)
+            # AI branding (stored in DB; blank value means "use framework default")
+            ident = (self.db.get_setting("branding.smxai_identity", "") or "").strip()
+            instr = (self.db.get_setting("branding.smxai_instructions", "") or "").strip()
+            wdesc = (self.db.get_setting("branding.website_description", "") or "").strip()
+
+            # Use the setters so behaviour is consistent across the framework.
+            self.set_smxai_identity(ident)
+            self.set_smxai_instructions(instr)
+            if wdesc:
+                self.set_website_description(wdesc)
+            else:
+                self.set_website_description(getattr(self, "_default_website_description", self.website_description))
 
         # Site title + project name (DB settings; fall back to defaults)
         try:
@@ -619,17 +704,20 @@ class SyntaxMUI:
     #  *********** LLM CLIENT HELPERS  **********************
     # ──────────────────────────────────────────────────────────────
     def set_smxai_identity(self, profile):
-        self.set_smxai_identity = profile
-    
+        """Set the system identity/profile used by the chat model."""
+        profile = (profile or "").strip()
+        self.smxai_identity = profile if profile else getattr(self, "_default_smxai_identity", self.smxai_identity)
 
     def set_smxai_instructions(self, instructions):
-        self.set_smxai_instructions = instructions
+        """Set additional system instructions used by the chat model."""
+        instructions = (instructions or "").strip()
+        self.smxai_instructions = instructions if instructions else getattr(self, "_default_smxai_instructions", self.smxai_instructions)
 
 
     def set_website_description(self, desc):
         self.website_description = desc
 
-
+    
     def embed_query(self, q):
         return embed_text(q)
     
@@ -705,7 +793,7 @@ class SyntaxMUI:
             self.classifier_profile['client'] = _prof.get_client(classifier_profile)
 
         _client = self.classifier_profile['client']
-        _provider = self.classifier_profile['provider']
+        _provider = self.classifier_profile['provider'].lower()
         _model = self.classifier_profile['model']
 
         # New instruction format with hybrid option
@@ -818,7 +906,7 @@ class SyntaxMUI:
         """
         
         _client = self.summarizer_profile['client']
-        _provider = self.summarizer_profile['provider']
+        _provider = self.summarizer_profile['provider'].lower()
         _model = self.summarizer_profile['model']
 
         def google_generated_title():
@@ -829,7 +917,7 @@ class SyntaxMUI:
                 )
                 return response.text.strip()
             except Exception as e:
-                return f"Google Summary agent error!"
+                return f"Google Summary agent error: {e}"
 
         def gpt_models_latest_generated_title():
             try:                 
@@ -905,16 +993,15 @@ class SyntaxMUI:
             if not chat_profile:
                 yield """
                 <p style='color:red;'>
-                    Error!<br> 
-                    Chat profile is not configured. Add a chat profile inside the admin panel.
-                    To do that, you must login first or contact your administrator.
+                    Error, Chat profile is not configured!
+                    Login to the admin panel and add the LLM profile for chatting or contact your administrator.
                 </p>
                 """
                 return None
             self.chat_profile = chat_profile
             self.chat_profile['client'] = _prof.get_client(chat_profile)
 
-        _provider = self.chat_profile['provider']
+        _provider = self.chat_profile['provider'].lower()
         _client = self.chat_profile['client']
         _model = self.chat_profile['model']
 
@@ -927,19 +1014,14 @@ class SyntaxMUI:
         """       
         
         try:
-            if _provider == "google":     # Google, non openai skd series     
-            
-                for chunk in _client.models.generate_content_stream(
+            if _provider == "google":  
+                chuncks = _client.models.generate_content_stream(
                     model=_model,
                     contents=_contents,
-                    config=types.GenerateContentConfig(
-                        system_instruction=self.smxai_identity,
-                        temperature=0.3,
-                        max_output_tokens=1024,
-                    ),
-                ):
-            
-                    yield chunk.text
+                )
+                for chunk in chuncks:
+                    if chunk:
+                        yield chunk.text
         
             elif _provider == "openai" and _model in self.get_gpt_models_latest():  # GPt 5 series
                 input_prompt = (
@@ -986,7 +1068,7 @@ class SyntaxMUI:
                     if token:
                         yield token
         except Exception as e:
-            yield f"Error during streaming: {type(e).__name__}: {e}"
+            yield f"Error during streaming: CLIENT {_client}"  # {type(e).__name__}: {e}"
     
 
     def process_query(self, query, context, history, stream=False):
@@ -996,16 +1078,15 @@ class SyntaxMUI:
             if not chat_profile:
                 yield """
                 <p style='color:red;'>
-                    Error!<br> 
-                    Chat profile is not configured. Add a chat profile inside the admin panel.
-                    To do that, you must login first or contact your administrator.
+                    Error, Chat profile is not configured!
+                    Login to the admin panel and add the LLM profile for chatting or contact your administrator.
                 </p>
                 """
-            return None
+                return None
                  
             self.chat_profile = chat_profile
             self.chat_profile['client'] = _prof.get_client(chat_profile) 
-        _provider = self.chat_profile['provider']
+        _provider = self.chat_profile['provider'].lower()
         _client = self.chat_profile['client']
         _model = self.chat_profile['model']
         _contents = f"""
@@ -1557,7 +1638,12 @@ class SyntaxMUI:
         current_profile['client'] = _prof.get_client(current_profile)
         return current_profile
 
-    def run(self):
+    def run(self, browser: Optional[str] = None) -> None:
         url = f"http://{self.host}:{self.port}/"
-        webbrowser.open(url)
+        
+        if browser:
+            webbrowser.get(browser).open(url)
+        else:
+            webbrowser.open(url)
+
         self.app.run(host=self.host, port=self.port, debug=False)

syntaxmatrix/dataset_preprocessing.py

@@ -181,9 +181,9 @@ def _impute_for_analysis(df: pd.DataFrame) -> Tuple[pd.DataFrame, Dict[str, str]
 def ensure_cleaned_df(DATA_FOLDER: str, cleaned_folder: str, df: pd.DataFrame) -> pd.DataFrame:
     """
     Build (or reuse) an analysis-ready cleaned dataset and persist to:
-        f"{DATA_FOLDER}/{selected_dataset}/cleaned_df.csv"
+        f"{DATA_FOLDER}/{selected_dataset_processed}/cleaned_df.csv"
     Also writes a missingness audit:
-        f"{DATA_FOLDER}/{selected_dataset}/missingness.csv"
+        f"{DATA_FOLDER}/{selected_dataset_processed}/missingness.csv"
     Returns the cleaned frame. Does NOT mutate the provided df.
     """
     target_dir = os.path.join(DATA_FOLDER, cleaned_folder)

syntaxmatrix/db.py

@@ -628,23 +628,6 @@ def get_setting(key: str, default: str | None = None) -> str | None:
         conn.close()
 
 
-# ============================================================================
-# Optional DB backend override (Premium hook)
-#
-# Default behaviour (no env vars): SQLite stays in use exactly as before.
-#
-# To enable a premium backend (e.g. Postgres), set:
-#   SMX_DB_PROVIDER=postgres
-#
-# Optional:
-#   SMX_DB_BACKEND_MODULE=syntaxmatrix_premium.db_backends.postgres_backend
-#
-# The backend module should either expose:
-#   - install(target_globals: dict) -> None     (preferred)
-# OR export a compatible surface (functions/constants) that will be copied
-# into this module namespace.
-# ============================================================================
-
 def _smx_apply_optional_backend_override() -> None:
     import os
     import importlib