arthexis 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

core/views.py

@@ -6,7 +6,7 @@ import requests
 from django.contrib.admin.views.decorators import staff_member_required
 from django.contrib.auth import authenticate, login
 from django.http import Http404, JsonResponse
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import get_object_or_404, render, redirect
 from django.views.decorators.csrf import csrf_exempt
 from pathlib import Path
 import subprocess
@@ -24,6 +24,12 @@ def _append_log(path: Path, message: str) -> None:
         fh.write(message + "\n")
 
 
+def _clean_repo() -> None:
+    """Return the git repository to a clean state."""
+    subprocess.run(["git", "reset", "--hard"], check=False)
+    subprocess.run(["git", "clean", "-fd"], check=False)
+
+
 def _changelog_notes(version: str) -> str:
     path = Path("CHANGELOG.rst")
     if not path.exists():
@@ -45,6 +51,51 @@ def _step_check_pypi(release, ctx, log_path: Path) -> None:
     from . import release as release_utils
     from packaging.version import Version
 
+    if not release_utils._git_clean():
+        proc = subprocess.run(
+            ["git", "status", "--porcelain"],
+            capture_output=True,
+            text=True,
+        )
+        files = [line[3:] for line in proc.stdout.splitlines()]
+        fixture_files = [
+            f
+            for f in files
+            if "fixtures" in Path(f).parts and Path(f).suffix == ".json"
+        ]
+        if not files or len(fixture_files) != len(files):
+            raise Exception("Git repository is not clean")
+
+        summary = []
+        for f in fixture_files:
+            path = Path(f)
+            try:
+                data = json.loads(path.read_text(encoding="utf-8"))
+            except Exception:
+                count = 0
+                models: list[str] = []
+            else:
+                if isinstance(data, list):
+                    count = len(data)
+                    models = sorted(
+                        {obj.get("model", "") for obj in data if isinstance(obj, dict)}
+                    )
+                elif isinstance(data, dict):
+                    count = 1
+                    models = [data.get("model", "")]
+                else:  # pragma: no cover - unexpected structure
+                    count = 0
+                    models = []
+            summary.append({"path": f, "count": count, "models": models})
+
+        ctx["fixtures"] = summary
+        _append_log(
+            log_path,
+            "Committing fixture changes: " + ", ".join(fixture_files),
+        )
+        subprocess.run(["git", "add", *fixture_files], check=True)
+        subprocess.run(["git", "commit", "-m", "chore: update fixtures"], check=True)
+
     version_path = Path("VERSION")
     if version_path.exists():
         current = version_path.read_text(encoding="utf-8").strip()
@@ -52,7 +103,6 @@ def _step_check_pypi(release, ctx, log_path: Path) -> None:
             raise Exception(
                 f"Version {release.version} is older than existing {current}"
             )
-    version_path.write_text(release.version + "\n", encoding="utf-8")
 
     _append_log(log_path, f"Checking if version {release.version} exists on PyPI")
     if release_utils.network_available():
@@ -75,141 +125,58 @@ def _step_check_pypi(release, ctx, log_path: Path) -> None:
 
 def _step_promote_build(release, ctx, log_path: Path) -> None:
     from . import release as release_utils
-    release.pypi_url = f"https://pypi.org/project/{release.package.name}/{release.version}/"
-    release.save(update_fields=["pypi_url"])
-    PackageRelease.dump_fixture()
     _append_log(log_path, "Generating build files")
-    commit_hash, branch, _current = release_utils.promote(
-        package=release.to_package(),
-        version=release.version,
-        creds=release.to_credentials(),
-    )
-    release.revision = commit_hash
-    release.save(update_fields=["revision"])
-    ctx["branch"] = branch
-    release_name = f"{release.package.name}-{release.version}-{commit_hash[:7]}"
-    new_log = log_path.with_name(f"{release_name}.log")
-    log_path.rename(new_log)
-    ctx["log"] = new_log.name
-    _append_log(new_log, "Build complete")
-
-
-def _step_push_branch(release, ctx, log_path: Path) -> None:
-    branch = ctx.get("branch")
-    _append_log(log_path, f"Pushing branch {branch}")
-    subprocess.run(["git", "push", "-u", "origin", branch], check=True)
-    pr_url = None
-    gh_path = shutil.which("gh")
-    if gh_path:
+    try:
         try:
-            title = f"Release candidate for {release.version}"
-            body = _changelog_notes(release.version)
-            proc = subprocess.run(
+            subprocess.run(["git", "fetch", "origin", "main"], check=True)
+            subprocess.run(["git", "rebase", "origin/main"], check=True)
+        except subprocess.CalledProcessError as exc:
+            subprocess.run(["git", "rebase", "--abort"], check=False)
+            raise Exception("Rebase onto main failed") from exc
+        release_utils.promote(
+            package=release.to_package(),
+            version=release.version,
+            creds=release.to_credentials(),
+        )
+        diff = subprocess.run(
+            [
+                "git",
+                "status",
+                "--porcelain",
+                "VERSION",
+                "core/fixtures/releases.json",
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if diff.stdout.strip():
+            subprocess.run(
+                ["git", "add", "VERSION", "core/fixtures/releases.json"],
+                check=True,
+            )
+            subprocess.run(
                 [
-                    gh_path,
-                    "pr",
-                    "create",
-                    "--title",
-                    title,
-                    "--body",
-                    body,
-                    "--base",
-                    "main",
-                    "--head",
-                    branch,
+                    "git",
+                    "commit",
+                    "-m",
+                    f"chore: update release metadata for v{release.version}",
                 ],
                 check=True,
-                capture_output=True,
-                text=True,
-            )
-            pr_url = proc.stdout.strip()
-            ctx["pr_url"] = pr_url
-            release.pr_url = pr_url
-            release.save(update_fields=["pr_url"])
-            _append_log(log_path, f"PR created: {pr_url}")
-            cert_log = Path("logs") / "certifications.log"
-            _append_log(cert_log, f"{release.version} {branch} {pr_url}")
-            ctx["cert_log"] = str(cert_log)
-        except Exception as exc:  # pragma: no cover - best effort
-            _append_log(log_path, f"PR creation failed: {exc}")
-    else:
-        token = release.get_github_token()
-        if token:
-            try:  # pragma: no cover - best effort
-                remote = subprocess.run(
-                    ["git", "config", "--get", "remote.origin.url"],
-                    check=True,
-                    capture_output=True,
-                    text=True,
-                ).stdout.strip()
-                repo = remote.rsplit(":", 1)[-1].split("github.com/")[-1].removesuffix(".git")
-                title = f"Release candidate for {release.version}"
-                body = _changelog_notes(release.version)
-                resp = requests.post(
-                    f"https://api.github.com/repos/{repo}/pulls",
-                    json={
-                        "title": title,
-                        "head": branch,
-                        "base": "main",
-                        "body": body,
-                    },
-                    headers={"Authorization": f"token {token}"},
-                    timeout=10,
-                )
-                resp.raise_for_status()
-                pr_url = resp.json().get("html_url")
-                if pr_url:
-                    ctx["pr_url"] = pr_url
-                    release.pr_url = pr_url
-                    release.save(update_fields=["pr_url"])
-                    _append_log(log_path, f"PR created: {pr_url}")
-                    cert_log = Path("logs") / "certifications.log"
-                    _append_log(cert_log, f"{release.version} {branch} {pr_url}")
-                    ctx["cert_log"] = str(cert_log)
-                else:
-                    _append_log(log_path, "PR creation failed: no URL returned")
-            except Exception as exc:
-                _append_log(log_path, f"PR creation failed: {exc}")
-        else:
-            _append_log(
-                log_path,
-                "PR creation skipped: gh not installed and no GitHub token available",
             )
-    subprocess.run(["git", "checkout", "main"], check=True)
-    _append_log(log_path, "Branch pushed")
+        subprocess.run(["git", "push"], check=True)
+        PackageRelease.dump_fixture()
+    except Exception:
+        _clean_repo()
+        raise
+    release_name = f"{release.package.name}-{release.version}"
+    new_log = log_path.with_name(f"{release_name}.log")
+    log_path.rename(new_log)
+    ctx["log"] = new_log.name
+    _append_log(new_log, "Build complete")
 
 
-def _step_merge_publish(release, ctx, log_path: Path) -> None:
+def _step_publish(release, ctx, log_path: Path) -> None:
     from . import release as release_utils
-    import time
-
-    gh_path = shutil.which("gh")
-    pr_url = ctx.get("pr_url") or release.pr_url
-    if gh_path and pr_url:
-        _append_log(log_path, "Waiting for PR checks")
-        for _ in range(60):
-            try:
-                proc = subprocess.run(
-                    [gh_path, "pr", "view", pr_url, "--json", "mergeable"],
-                    capture_output=True,
-                    text=True,
-                    check=True,
-                )
-                state = json.loads(proc.stdout or "{}").get("mergeable")
-                if state == "MERGEABLE":
-                    break
-            except Exception:
-                pass
-            time.sleep(1)
-        _append_log(log_path, "Merging PR")
-        try:
-            subprocess.run(
-                [gh_path, "pr", "merge", pr_url, "--merge", "--delete-branch"],
-                check=True,
-            )
-            subprocess.run(["git", "pull", "--ff-only", "origin", "main"], check=True)
-        except Exception as exc:
-            _append_log(log_path, f"PR merge failed: {exc}")
 
     _append_log(log_path, "Uploading distribution")
     release_utils.publish(
@@ -217,14 +184,16 @@ def _step_merge_publish(release, ctx, log_path: Path) -> None:
         version=release.version,
         creds=release.to_credentials(),
     )
+    release.pypi_url = f"https://pypi.org/project/{release.package.name}/{release.version}/"
+    release.save(update_fields=["pypi_url"])
+    PackageRelease.dump_fixture()
     _append_log(log_path, "Upload complete")
 
 
 PUBLISH_STEPS = [
     ("Check version availability", _step_check_pypi),
     ("Generate build", _step_promote_build),
-    ("Push branch", _step_push_branch),
-    ("Merge and publish", _step_merge_publish),
+    ("Publish", _step_publish),
 ]
 
 
@@ -392,13 +361,48 @@ def release_progress(request, pk: int, action: str):
     if action != "publish":
         raise Http404("Unknown action")
     session_key = f"release_publish_{pk}"
-    ctx = request.session.get(session_key, {"step": 0})
+    lock_path = Path("locks") / f"release_publish_{pk}.json"
+    restart_path = Path("locks") / f"release_publish_{pk}.restarts"
+
+    if request.GET.get("restart"):
+        count = 0
+        if restart_path.exists():
+            try:
+                count = int(restart_path.read_text(encoding="utf-8"))
+            except Exception:
+                count = 0
+        restart_path.parent.mkdir(parents=True, exist_ok=True)
+        restart_path.write_text(str(count + 1), encoding="utf-8")
+        _clean_repo()
+        release.pypi_url = ""
+        release.save(update_fields=["pypi_url"])
+        request.session.pop(session_key, None)
+        if lock_path.exists():
+            lock_path.unlink()
+        log_dir = Path("logs")
+        for f in log_dir.glob(f"{release.package.name}-{release.version}*.log"):
+            f.unlink()
+        return redirect(request.path)
+    ctx = request.session.get(session_key)
+    if ctx is None and lock_path.exists():
+        try:
+            ctx = json.loads(lock_path.read_text(encoding="utf-8"))
+        except Exception:
+            ctx = {"step": 0}
+    if ctx is None:
+        ctx = {"step": 0}
+        if restart_path.exists():
+            restart_path.unlink()
+    restart_count = 0
+    if restart_path.exists():
+        try:
+            restart_count = int(restart_path.read_text(encoding="utf-8"))
+        except Exception:
+            restart_count = 0
     step_count = ctx.get("step", 0)
     step_param = request.GET.get("step")
 
     identifier = f"{release.package.name}-{release.version}"
-    if release.revision:
-        identifier = f"{identifier}-{release.revision[:7]}"
     log_name = f"{identifier}.log"
     if ctx.get("log") != log_name:
         ctx = {"step": 0, "log": log_name}
@@ -406,6 +410,10 @@ def release_progress(request, pk: int, action: str):
     log_path = Path("logs") / log_name
     ctx.setdefault("log", log_name)
 
+    if step_count == 0 and (step_param is None or step_param == "0"):
+        if log_path.exists():
+            log_path.unlink()
+
     steps = PUBLISH_STEPS
     error = ctx.get("error")
 
@@ -418,10 +426,14 @@ def release_progress(request, pk: int, action: str):
                 step_count += 1
                 ctx["step"] = step_count
                 request.session[session_key] = ctx
+                lock_path.parent.mkdir(parents=True, exist_ok=True)
+                lock_path.write_text(json.dumps(ctx), encoding="utf-8")
             except Exception as exc:  # pragma: no cover - best effort logging
                 _append_log(log_path, f"{name} failed: {exc}")
                 ctx["error"] = str(exc)
                 request.session[session_key] = ctx
+                lock_path.parent.mkdir(parents=True, exist_ok=True)
+                lock_path.write_text(json.dumps(ctx), encoding="utf-8")
 
     done = step_count >= len(steps) and not ctx.get("error")
 
@@ -437,8 +449,15 @@ def release_progress(request, pk: int, action: str):
         "error": ctx.get("error"),
         "log_content": log_content,
         "log_path": str(log_path),
-        "pr_url": ctx.get("pr_url"),
         "cert_log": ctx.get("cert_log"),
+        "fixtures": ctx.get("fixtures"),
+        "restart_count": restart_count,
     }
     request.session[session_key] = ctx
+    if done or ctx.get("error"):
+        if lock_path.exists():
+            lock_path.unlink()
+    else:
+        lock_path.parent.mkdir(parents=True, exist_ok=True)
+        lock_path.write_text(json.dumps(ctx), encoding="utf-8")
     return render(request, "core/release_progress.html", context)

nodes/admin.py

@@ -9,10 +9,16 @@ from django.db import models
 from django.conf import settings
 from pathlib import Path
 from django.http import HttpResponse
+from django.template.response import TemplateResponse
+from django.core.management import call_command
 import base64
 import pyperclip
 from pyperclip import PyperclipException
 import uuid
+import subprocess
+import io
+import threading
+import re
 from .utils import capture_screenshot, save_screenshot
 from .actions import NodeAction
 
@@ -23,11 +29,18 @@ from .models import (
     ContentSample,
     NodeTask,
     NetMessage,
+    Operation,
+    Interrupt,
+    Logbook,
     User,
 )
 from core.admin import UserAdmin as CoreUserAdmin
 
 
+RUN_CONTEXTS: dict[int, dict] = {}
+SIGIL_RE = re.compile(r"\[[A-Za-z0-9_]+\.[A-Za-z0-9_]+\]")
+
+
 class NodeAdminForm(forms.ModelForm):
     class Meta:
         model = Node
@@ -356,4 +369,202 @@ class NodeTaskAdmin(admin.ModelAdmin):
     execute.short_description = "Run task on nodes"
 
 
+@admin.register(Operation)
+class OperationAdmin(admin.ModelAdmin):
+    list_display = ("name",)
+    formfield_overrides = {models.TextField: {"widget": CodeEditorWidget}}
+    change_form_template = "admin/nodes/operation/change_form.html"
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom = [
+            path(
+                "<path:object_id>/run/",
+                self.admin_site.admin_view(self.run_view),
+                name="nodes_operation_run",
+            )
+        ]
+        return custom + urls
+
+    def run_view(self, request, object_id):
+        operation = self.get_object(request, object_id)
+        if not operation:
+            self.message_user(request, "Unknown operation", messages.ERROR)
+            return redirect("..")
+        context = RUN_CONTEXTS.setdefault(operation.pk, {"inputs": {}})
+        template_text = operation.resolve_sigils("template")
+
+        # Interrupt handling
+        interrupt_id = request.GET.get("interrupt")
+        if interrupt_id:
+            try:
+                interrupt = operation.outgoing_interrupts.get(pk=interrupt_id)
+            except Interrupt.DoesNotExist:
+                self.message_user(request, "Unknown interrupt", messages.ERROR)
+                return redirect(request.path)
+            proc = context.get("process")
+            if proc and proc.poll() is None:
+                proc.terminate()
+                out, err = proc.communicate()
+                log = context.get("log")
+                if log:
+                    log.output = out
+                    log.error = err
+                    log.interrupted = True
+                    log.interrupt = interrupt
+                    log.save()
+            RUN_CONTEXTS.pop(operation.pk, None)
+            return redirect(
+                reverse("admin:nodes_operation_run", args=[interrupt.to_operation.pk])
+            )
+
+        # Check running processes
+        proc = context.get("process")
+        thread = context.get("thread")
+        if proc and proc.poll() is not None:
+            out, err = proc.communicate()
+            log = context.pop("log")
+            log.output = out
+            log.error = err
+            log.save()
+            RUN_CONTEXTS.pop(operation.pk, None)
+            self.message_user(request, "Operation executed", messages.SUCCESS)
+            return redirect("..")
+        if thread and not thread.is_alive():
+            out = context.pop("out")
+            err = context.pop("err")
+            log = context.pop("log")
+            log.output = out.getvalue()
+            log.error = err.getvalue()
+            log.save()
+            RUN_CONTEXTS.pop(operation.pk, None)
+            self.message_user(request, "Operation executed", messages.SUCCESS)
+            return redirect("..")
+
+        interrupts = [
+            (i, i.resolve_sigils("preview"))
+            for i in operation.outgoing_interrupts.all().order_by("-priority")
+        ]
+        logs = Logbook.objects.filter(operation=operation).order_by("created")
+
+        # Waiting for user-provided sigils
+        waiting = context.get("waiting_inputs")
+        if waiting:
+            if request.method == "POST":
+                for token in waiting:
+                    name = token[1:-1].replace(".", "__")
+                    context["inputs"][token] = request.POST.get(name, "")
+                command = context.pop("pending_command")
+                for token, value in context["inputs"].items():
+                    command = command.replace(token, value)
+                context["waiting_inputs"] = None
+                self._start_operation(context, operation, command, request.user)
+                return redirect(request.path)
+            form_fields = [(t, t[1:-1].replace(".", "__")) for t in waiting]
+            tpl_context = {
+                **self.admin_site.each_context(request),
+                "operation": operation,
+                "interrupts": interrupts,
+                "logs": logs,
+                "waiting_inputs": form_fields,
+                "template": template_text,
+            }
+            return TemplateResponse(
+                request, "admin/nodes/operation/run.html", tpl_context
+            )
+
+        # Waiting for user continuation
+        if context.get("waiting_continue"):
+            if request.method == "POST":
+                context["waiting_continue"] = False
+                RUN_CONTEXTS.pop(operation.pk, None)
+                self.message_user(request, "Operation executed", messages.SUCCESS)
+                return redirect("..")
+            tpl_context = {
+                **self.admin_site.each_context(request),
+                "operation": operation,
+                "interrupts": interrupts,
+                "logs": logs,
+                "waiting_continue": True,
+                "template": template_text,
+            }
+            return TemplateResponse(
+                request, "admin/nodes/operation/run.html", tpl_context
+            )
+
+        # If a process or thread is running, show running state
+        if context.get("process") or context.get("thread"):
+            tpl_context = {
+                **self.admin_site.each_context(request),
+                "operation": operation,
+                "interrupts": interrupts,
+                "logs": logs,
+                "running": True,
+                "template": template_text,
+            }
+            return TemplateResponse(
+                request, "admin/nodes/operation/run.html", tpl_context
+            )
+
+        if request.method == "POST":
+            command = operation.resolve_sigils("command")
+            for token, value in context["inputs"].items():
+                command = command.replace(token, value)
+            unresolved = SIGIL_RE.findall(command)
+            if unresolved:
+                context["waiting_inputs"] = unresolved
+                context["pending_command"] = command
+                return redirect(request.path)
+            if command.strip() == "...":
+                log = Logbook.objects.create(
+                    operation=operation,
+                    user=request.user,
+                    input_text=command,
+                    output="Waiting for user continuation",
+                )
+                context["log"] = log
+                context["waiting_continue"] = True
+                return redirect(request.path)
+            self._start_operation(context, operation, command, request.user)
+            return redirect(request.path)
+
+        tpl_context = {
+            **self.admin_site.each_context(request),
+            "operation": operation,
+            "interrupts": interrupts,
+            "logs": logs,
+            "template": template_text,
+        }
+        return TemplateResponse(request, "admin/nodes/operation/run.html", tpl_context)
+
+    def _start_operation(self, ctx, operation, command, user):
+        log = Logbook.objects.create(operation=operation, user=user, input_text=command)
+        if operation.is_django:
+            out = io.StringIO()
+            err = io.StringIO()
+
+            def target():
+                try:
+                    call_command(*command.split(), stdout=out, stderr=err)
+                except Exception as exc:  # pragma: no cover - unexpected errors
+                    err.write(str(exc))
+
+            thread = threading.Thread(target=target)
+            thread.start()
+            ctx.update({"thread": thread, "out": out, "err": err, "log": log})
+        else:
+            proc = subprocess.Popen(
+                command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+            ctx.update({"process": proc, "log": log})
+
+    def changeform_view(self, request, object_id=None, form_url="", extra_context=None):
+        extra_context = extra_context or {}
+        if object_id:
+            extra_context["run_url"] = reverse(
+                "admin:nodes_operation_run", args=[object_id]
+            )
+        return super().changeform_view(request, object_id, form_url, extra_context)
+
+
 admin.site.register(User, CoreUserAdmin)

nodes/apps.py

@@ -68,7 +68,7 @@ def _trigger_startup_notification(**_: object) -> None:
 class NodesConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "nodes"
-    verbose_name = "Node Infrastructure"
+    verbose_name = "2. Infrastructure"
 
     def ready(self):  # pragma: no cover - exercised on app start
         request_started.connect(