qalita 2.3.1__py3-none-any.whl → 2.5.2__py3-none-any.whl

qalita/web/blueprints/helpers.py

@@ -7,11 +7,38 @@ import re
 import sys
 import shutil
 import subprocess
+import threading
 from datetime import datetime
 from flask import current_app
 
 from qalita.internal.utils import logger
 
+# Lock for .current_env file operations
+_current_env_lock = threading.Lock()
+
+
+def write_selected_env_atomic(pointer_path: str, env_path: str) -> bool:
+    """Atomically write the selected env pointer file. Uses locking to prevent race conditions.
+
+    Args:
+        pointer_path: Path to the .current_env pointer file
+        env_path: Path to the env file to write in the pointer
+
+    Returns:
+        True if successful, False otherwise
+    """
+    with _current_env_lock:
+        try:
+            # Use atomic write: write to temp file then rename
+            temp_path = pointer_path + ".tmp"
+            with open(temp_path, "w", encoding="utf-8") as pf:
+                pf.write(env_path)
+            os.replace(temp_path, pointer_path)
+            return True
+        except Exception as exc:
+            logger.warning(f"Failed to write selected env pointer: {exc}")
+            return False
+
 
 def qalita_home() -> str:
     try:
@@ -44,30 +71,58 @@ def parse_env_file(env_path: str) -> dict:
 
 
 def materialize_env_from_process_env(target_path: str) -> None:
+    """Update specific keys in an env file from process environment variables.
+
+    This function preserves ALL existing keys in the file and only updates
+    specific keys (NAME, MODE, TOKEN, URL) from the process environment.
+    It should only be called when creating a NEW file, not on existing files
+    that may contain important configuration.
+    """
     try:
         existing: dict[str, str] = {}
         if os.path.isfile(target_path):
             existing = parse_env_file(target_path) or {}
         env = os.environ
         key_groups = [
-            ("QALITA_AGENT_NAME", ["QALITA_AGENT_NAME", "AGENT_NAME", "NAME"]),
-            ("QALITA_AGENT_MODE", ["QALITA_AGENT_MODE", "AGENT_MODE", "MODE"]),
-            ("QALITA_AGENT_TOKEN", ["QALITA_AGENT_TOKEN", "QALITA_TOKEN", "TOKEN"]),
+            ("QALITA_WORKER_NAME", ["QALITA_WORKER_NAME", "QALITA_AGENT_NAME", "AGENT_NAME", "NAME"]),
+            ("QALITA_WORKER_MODE", ["QALITA_WORKER_MODE", "QALITA_AGENT_MODE", "AGENT_MODE", "MODE"]),
+            ("QALITA_WORKER_TOKEN", ["QALITA_WORKER_TOKEN", "QALITA_AGENT_TOKEN", "QALITA_TOKEN", "TOKEN"]),
             (
-                "QALITA_AGENT_ENDPOINT",
-                ["QALITA_AGENT_ENDPOINT", "AGENT_ENDPOINT", "QALITA_URL", "URL"],
+                "QALITA_WORKER_ENDPOINT",
+                ["QALITA_WORKER_ENDPOINT", "QALITA_AGENT_ENDPOINT", "AGENT_ENDPOINT", "QALITA_URL", "URL"],
             ),
         ]
-        updates: dict[str, str] = {}
-        for _, aliases in key_groups:
-            value = None
-            for k in aliases:
-                if k in env and env.get(k):
-                    value = env.get(k)
-                    updates[k] = value  # type: ignore[assignment]
-                    break
+        # Start with existing keys to preserve everything
         merged = dict(existing)
-        merged.update(updates)
+
+        # Only update specific keys from environment if they exist
+        # Map environment variable names to their canonical key names (use WORKER as canonical)
+        canonical_keys = {
+            "QALITA_WORKER_NAME": "QALITA_WORKER_NAME",
+            "QALITA_AGENT_NAME": "QALITA_WORKER_NAME",
+            "AGENT_NAME": "QALITA_WORKER_NAME",
+            "NAME": "QALITA_WORKER_NAME",
+            "QALITA_WORKER_MODE": "QALITA_WORKER_MODE",
+            "QALITA_AGENT_MODE": "QALITA_WORKER_MODE",
+            "AGENT_MODE": "QALITA_WORKER_MODE",
+            "MODE": "QALITA_WORKER_MODE",
+            "QALITA_WORKER_TOKEN": "QALITA_WORKER_TOKEN",
+            "QALITA_AGENT_TOKEN": "QALITA_WORKER_TOKEN",
+            "QALITA_TOKEN": "QALITA_WORKER_TOKEN",
+            "TOKEN": "QALITA_WORKER_TOKEN",
+            "QALITA_WORKER_ENDPOINT": "QALITA_WORKER_ENDPOINT",
+            "QALITA_AGENT_ENDPOINT": "QALITA_WORKER_ENDPOINT",
+            "AGENT_ENDPOINT": "QALITA_WORKER_ENDPOINT",
+            "QALITA_URL": "QALITA_WORKER_ENDPOINT",
+            "URL": "QALITA_WORKER_ENDPOINT",
+        }
+
+        # Update only the canonical keys from environment
+        for env_key, canonical_key in canonical_keys.items():
+            if env_key in env and env.get(env_key):
+                merged[canonical_key] = env.get(env_key)
+
+        # Write all keys (existing + updated)
         lines = []
         for k in sorted(merged.keys()):
             v = merged[k]
@@ -87,6 +142,17 @@ def materialize_env_from_process_env(target_path: str) -> None:
 
 
 def ensure_default_env_selected(pointer_path: str):
+    """Ensure a default env is selected. Uses locking to prevent race conditions."""
+    # Double-check: if file was created by another thread, use it
+    try:
+        if os.path.isfile(pointer_path):
+            with open(pointer_path, "r", encoding="utf-8") as f:
+                existing_path = f.read().strip()
+                if existing_path and os.path.isfile(existing_path):
+                    return existing_path
+    except Exception:
+        pass
+
     try:
         base = qalita_home()
         env = os.environ
@@ -103,65 +169,80 @@ def ensure_default_env_selected(pointer_path: str):
         target = os.path.normpath(os.path.join(base, f".env-{safe}"))
         os.makedirs(base, exist_ok=True)
         materialize_env_from_process_env(target)
-        try:
-            with open(pointer_path, "w", encoding="utf-8") as pf:
-                pf.write(target)
-        except Exception:
-            pass
+        # Use atomic write function (which handles locking)
+        write_selected_env_atomic(pointer_path, target)
         return target
     except Exception:
         return None
 
 
 def read_selected_env():
+    """Read the selected env file path. Uses locking to prevent race conditions when creating default.
+
+    NOTE: Does NOT call materialize_env_from_process_env() to avoid overwriting existing .env files.
+    The materialization should only happen when explicitly creating a new file, not on every read.
+    """
     p = selected_env_file_path()
     try:
-        with open(p, "r", encoding="utf-8") as f:
-            raw = f.read().strip()
-            path = os.path.normpath(raw) if raw else None
-            if path and os.path.isfile(path):
+        # Try to read the file first (read-only, no lock needed)
+        if os.path.isfile(p):
+            with open(p, "r", encoding="utf-8") as f:
+                raw = f.read().strip()
+                path = os.path.normpath(raw) if raw else None
+                if path and os.path.isfile(path):
+                    # DO NOT call materialize_env_from_process_env() here - it would overwrite the file!
+                    # Only return the path without modifying the file
+                    return path
+                # Path exists but file doesn't - try to find it
                 try:
-                    materialize_env_from_process_env(path)
-                except Exception:
-                    pass
-                return path
-            try:
-                base = qalita_home()
-                if path:
-                    candidate = os.path.normpath(
-                        os.path.join(base, os.path.basename(path))
-                    )
-                    if os.path.isfile(candidate):
-                        logger.warning(
-                            f"Selected env pointer [{path}] not found. Using [{candidate}] under current QALITA_HOME."
+                    base = qalita_home()
+                    if path:
+                        candidate = os.path.normpath(
+                            os.path.join(base, os.path.basename(path))
                         )
+                        if os.path.isfile(candidate):
+                            logger.warning(
+                                f"Selected env pointer [{path}] not found. Using [{candidate}] under current QALITA_HOME."
+                            )
+                            # DO NOT call materialize_env_from_process_env() - preserve existing file content
+                            # Use atomic write function (which handles locking)
+                            # Double-check: if file was updated by another thread, use it
+                            try:
+                                if os.path.isfile(p):
+                                    with open(p, "r", encoding="utf-8") as f2:
+                                        existing = f2.read().strip()
+                                        if existing and os.path.isfile(existing):
+                                            return existing
+                            except Exception:
+                                pass
+                            write_selected_env_atomic(p, candidate)
+                            return candidate
                         try:
+                            os.makedirs(base, exist_ok=True)
+                            # Only call materialize_env_from_process_env when creating a NEW file
+                            # This is safe because the file doesn't exist yet
                             materialize_env_from_process_env(candidate)
+                            logger.warning(
+                                f"Selected env pointer [{path}] not found. Created [{candidate}] under current QALITA_HOME from environment."
+                            )
+                            # Use atomic write function (which handles locking)
+                            # Double-check: if file was updated by another thread, use it
+                            try:
+                                if os.path.isfile(p):
+                                    with open(p, "r", encoding="utf-8") as f2:
+                                        existing = f2.read().strip()
+                                        if existing and os.path.isfile(existing):
+                                            return existing
+                            except Exception:
+                                pass
+                            write_selected_env_atomic(p, candidate)
+                            return candidate
                         except Exception:
                             pass
-                        try:
-                            with open(p, "w", encoding="utf-8") as pf:
-                                pf.write(candidate)
-                        except Exception:
-                            pass
-                        return candidate
-                    try:
-                        os.makedirs(base, exist_ok=True)
-                        materialize_env_from_process_env(candidate)
-                        logger.warning(
-                            f"Selected env pointer [{path}] not found. Created [{candidate}] under current QALITA_HOME from environment."
-                        )
-                        try:
-                            with open(p, "w", encoding="utf-8") as pf:
-                                pf.write(candidate)
-                        except Exception:
-                            pass
-                        return candidate
-                    except Exception:
-                        pass
-            except Exception:
-                pass
-            return ensure_default_env_selected(p)
+                except Exception:
+                    pass
+        # File doesn't exist or is invalid - create default (with lock)
+        return ensure_default_env_selected(p)
     except Exception:
         logger.warning(f"No selected env pointer found at [{p}] or failed to read it")
         return ensure_default_env_selected(p)
@@ -283,7 +364,7 @@ def compute_agent_summary(cfg):
 
             agent_conf = {
                 "name": pick(raw, "context", "remote", "name") or raw.get("name", ""),
-                "mode": raw.get("mode", ""),
+                "mode": pick(raw, "context", "local", "mode") or raw.get("mode", ""),
                 "url": pick(raw, "context", "local", "url") or raw.get("url", ""),
                 "agent_id": pick(raw, "context", "remote", "id")
                 or raw.get("agent_id", ""),
@@ -303,19 +384,22 @@ def compute_agent_summary(cfg):
             if agent_conf is None:
                 agent_conf = {}
             agent_conf["name"] = (
-                data.get("QALITA_AGENT_NAME")
+                data.get("QALITA_WORKER_NAME")
+                or data.get("QALITA_AGENT_NAME")
                 or data.get("AGENT_NAME")
                 or data.get("NAME")
                 or agent_conf.get("name", "")
             )
             agent_conf["mode"] = (
-                data.get("QALITA_AGENT_MODE")
+                data.get("QALITA_WORKER_MODE")
+                or data.get("QALITA_AGENT_MODE")
                 or data.get("AGENT_MODE")
                 or data.get("MODE")
                 or agent_conf.get("mode", "")
             )
             agent_conf["url"] = (
-                data.get("QALITA_AGENT_ENDPOINT")
+                data.get("QALITA_WORKER_ENDPOINT")
+                or data.get("QALITA_AGENT_ENDPOINT")
                 or data.get("QALITA_URL")
                 or data.get("URL")
                 or agent_conf.get("url", "")
@@ -362,3 +446,10 @@ def compute_agent_summary(cfg):
     except Exception:
         agent_runs = []
     return agent_conf, agent_runs
+
+
+# Aliases for backward compatibility (agent -> worker refactoring)
+worker_pid_file_path = agent_pid_file_path
+worker_log_file_path = agent_log_file_path
+worker_status_payload = agent_status_payload
+compute_worker_summary = compute_agent_summary

qalita/web/blueprints/sources.py

@@ -13,10 +13,7 @@ from qalita.internal.utils import logger
 from flask import (
     Blueprint,
     current_app,
-    redirect,
-    render_template,
     request,
-    url_for,
     jsonify,
 )
 from qalita.internal.request import send_request
@@ -88,37 +85,10 @@ def list_sources():
     if isinstance(platform_url, str):
         platform_url = platform_url.rstrip("/")
 
-    return render_template(
-        "sources/list.html", sources=sources, platform_url=platform_url
-    )
-
-
-@bp.get("/add")
-def add_source_view():
-    # If a valid type is provided as query param, go directly to the form with that type preselected.
-    # Otherwise, render the selection grid first.
-    t = (request.args.get("type") or "").strip().lower()
-    allowed = [
-        "file",
-        "csv",
-        "excel",
-        "folder",
-        "postgresql",
-        "mysql",
-        "oracle",
-        "mssql",
-        "sqlite",
-        "mongodb",
-        "s3",
-        "gcs",
-        "azure_blob",
-        "hdfs",
-    ]
-    if t and t in allowed:
-        return render_template(
-            "sources/edit.html", title="Add source", source=None, preselected_type=t
-        )
-    return render_template("sources/select-source.html")
+    return jsonify({
+        "sources": sources,
+        "platform_url": platform_url,
+    })
 
 
 @bp.post("/add")
@@ -245,12 +215,10 @@ def add_source_post():
     }
 
     if not validate_source_object(cfg, new_source, skip_connection=False):
-        return render_template(
-            "sources/edit.html",
-            title="Add source",
-            source=request.form,
-            error="Validation failed. Check fields and connectivity.",
-        )
+        return jsonify({
+            "ok": False,
+            "error": "Validation failed. Check fields and connectivity.",
+        }), 400
 
     cfg.config.setdefault("sources", []).append(new_source)
     cfg.save_source_config()
@@ -329,19 +297,13 @@ def add_source_post():
     except Exception:
         src_id = None
 
-    return render_template(
-        "sources/added.html", name=name, platform_url=platform_url, source_id=src_id
-    )
-
-
-@bp.get("/edit/<name>")
-def edit_source_view(name):
-    cfg = current_app.config["QALITA_CONFIG_OBJ"]
-    cfg.load_source_config()
-    src = next(
-        (s for s in cfg.config.get("sources", []) if s.get("name") == name), None
-    )
-    return render_template("sources/edit.html", title="Edit source", source=src)
+    return jsonify({
+        "ok": True,
+        "name": name,
+        "platform_url": platform_url,
+        "source_id": src_id,
+        "message": f"Source '{name}' added successfully.",
+    })
 
 
 @bp.post("/edit/<name>")
@@ -482,12 +444,10 @@ def edit_source_post(name):
             if not validate_source_object(
                 cfg, updated, skip_connection=False, exclude_name=name
             ):
-                return render_template(
-                    "sources/edit.html",
-                    title="Edit source",
-                    source=request.form,
-                    error="Validation failed. Check fields and connectivity.",
-                )
+                return jsonify({
+                    "ok": False,
+                    "error": "Validation failed. Check fields and connectivity.",
+                }), 400
 
             sources[i].update(updated)
             break
@@ -504,7 +464,11 @@ def edit_source_post(name):
         push_single_programmatic(cfg, new_name, approve_public=False)
     except (SystemExit, Exception):
         pass
-    return redirect(url_for("dashboard.dashboard"))
+    return jsonify({
+        "ok": True,
+        "name": new_name,
+        "message": f"Source '{new_name}' updated successfully.",
+    })
 
 
 @bp.post("/delete/<name>")
@@ -515,7 +479,11 @@ def delete_source_post(name):
         s for s in cfg.config.get("sources", []) if s.get("name") != name
     ]
     cfg.save_source_config()
-    return redirect(url_for("dashboard.dashboard"))
+    return jsonify({
+        "ok": True,
+        "name": name,
+        "message": f"Source '{name}' deleted successfully.",
+    })
 
 
 @bp.get("/pick-file")