diracx 0.0.1a46__tar.gz → 0.0.1a47__tar.gz

{diracx-0.0.1a46 → diracx-0.0.1a47}/.github/workflows/integration.yml

@@ -58,11 +58,17 @@ jobs:
         run: |
           has_error=0
           # TODO: set has_error=1 when we are ready to really run the tests
-          if [ -f server-tests-failed ]; then has_error=0; echo "Server tests failed"; fi
-          if [ -f client-tests-failed ]; then has_error=0; echo "Client tests failed"; fi
-          if [ -f pilot-tests-failed ]; then has_error=0; echo "Pilot tests failed"; fi
+          if [ -f server-tests-failed ]; then has_error=1; echo "Server tests failed"; fi
+          if [ -f client-tests-failed ]; then has_error=1; echo "Client tests failed"; fi
+          if [ -f pilot-tests-failed ]; then has_error=1; echo "Pilot tests failed"; fi
           if [ ${has_error} = 1 ]; then exit 1; fi
-      - name: diracx logs
+      - name: diracx filtered requests logs
+        run: |
+          docker logs diracx 2>/dev/null | grep '/api/' \
+            | awk -F\" '{print $2, $3}' \
+            | awk '{print $1, $2, $4}' \
+            | sort | uniq -c | sort
+      - name: diracx error logs
         if: ${{ failure() }}
         run: |
           mkdir -p /tmp/service-logs

{diracx-0.0.1a46 → diracx-0.0.1a47}/.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           run-install: false
           post-cleanup: false
@@ -32,7 +32,7 @@ jobs:
           pixi add --no-install --pypi --feature diracx-core pytest-github-actions-annotate-failures
           # Show any changes
           git diff
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           cache: false
           environments: shellcheck
@@ -63,7 +63,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           run-install: false
           post-cleanup: false
@@ -76,7 +76,7 @@ jobs:
           pixi add --no-install --pypi --feature diracx-core pytest-github-actions-annotate-failures
           # Show any changes
           git diff
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           cache: false
           environments: ${{ matrix.package }}
@@ -96,7 +96,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           run-install: false
           post-cleanup: false
@@ -109,7 +109,7 @@ jobs:
           pixi add --no-install --pypi --feature diracx-core pytest-github-actions-annotate-failures
           # Show any changes
           git diff
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           cache: false
           environments: ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }}
@@ -222,7 +222,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           run-install: false
           post-cleanup: false
@@ -232,7 +232,7 @@ jobs:
           pixi add --no-install --pypi --feature diracx-core pytest-github-actions-annotate-failures
           # Show any changes
           git diff
-      - uses: prefix-dev/setup-pixi@v0.8.11
+      - uses: prefix-dev/setup-pixi@v0.8.12
         with:
           cache: false
           locked: false

{diracx-0.0.1a46 → diracx-0.0.1a47}/.github/workflows/update_security_txt_expiry.yml

@@ -19,7 +19,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 

{diracx-0.0.1a46 → diracx-0.0.1a47}/.pre-commit-config.yaml

@@ -27,14 +27,14 @@ repos:
       - id: check-added-large-files
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.12.3'
+    rev: 'v0.12.7'
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.16.1
+    rev: v1.17.1
     hooks:
       - id: mypy
         additional_dependencies:
@@ -53,6 +53,7 @@ repos:
     - id: mdformat
       args: ["--number"]
       additional_dependencies:
+      - mdformat-mkdocs
       - mdformat-gfm
       - mdformat-black
 

{diracx-0.0.1a46 → diracx-0.0.1a47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: diracx
-Version: 0.0.1a46
+Version: 0.0.1a47
 Summary: Client installation for users of DiracX installations
 License: GPL-3.0-only
 License-File: LICENSE

{diracx-0.0.1a46 → diracx-0.0.1a47}/diracx-api/pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "diracx-client",
     "diracx-core",
     "httpx",
+    "zstandard",
 ]
 dynamic = ["version"]
 

{diracx-0.0.1a46 → diracx-0.0.1a47}/diracx-api/src/diracx/api/jobs.py

@@ -7,10 +7,12 @@ import logging
 import os
 import tarfile
 import tempfile
+from contextlib import contextmanager
 from pathlib import Path
-from typing import Literal
+from typing import BinaryIO, Literal
 
 import httpx
+import zstandard
 
 from diracx.client.aio import AsyncDiracClient
 from diracx.client.models import SandboxInfo
@@ -20,8 +22,29 @@ from .utils import with_client
 logger = logging.getLogger(__name__)
 
 SANDBOX_CHECKSUM_ALGORITHM = "sha256"
-SANDBOX_COMPRESSION: Literal["bz2"] = "bz2"
-SANDBOX_OPEN_MODE: Literal["w|bz2"] = "w|bz2"
+SANDBOX_COMPRESSION: Literal["zst"] = "zst"
+
+
+@contextmanager
+def tarfile_open(fileobj: BinaryIO):
+    """Context manager to extend tarfile.open to support reading zstd compressed files.
+
+    This is only needed for Python <=3.13.
+    """
+    # Save current position and read magic bytes
+    current_pos = fileobj.tell()
+    magic = fileobj.read(4)
+    fileobj.seek(current_pos)
+
+    # Read magic bytes to determine compression format
+    if magic.startswith(b"\x28\xb5\x2f\xfd"):  # zstd magic number
+        dctx = zstandard.ZstdDecompressor()
+        with dctx.stream_reader(fileobj) as decompressor:
+            with tarfile.open(fileobj=decompressor, mode="r|") as tf:
+                yield tf
+    else:
+        with tarfile.open(fileobj=fileobj, mode="r") as tf:
+            yield tf
 
 
 @with_client
@@ -33,10 +56,18 @@ async def create_sandbox(paths: list[Path], *, client: AsyncDiracClient) -> str:
     be used to submit jobs.
     """
     with tempfile.TemporaryFile(mode="w+b") as tar_fh:
-        with tarfile.open(fileobj=tar_fh, mode=SANDBOX_OPEN_MODE) as tf:
-            for path in paths:
-                logger.debug("Adding %s to sandbox as %s", path.resolve(), path.name)
-                tf.add(path.resolve(), path.name, recursive=True)
+        # Create zstd compressed tar with level 18 and long matching enabled
+        compression_params = zstandard.ZstdCompressionParameters.from_level(
+            18, enable_ldm=1
+        )
+        cctx = zstandard.ZstdCompressor(compression_params=compression_params)
+        with cctx.stream_writer(tar_fh, closefd=False) as compressor:
+            with tarfile.open(fileobj=compressor, mode="w|") as tf:
+                for path in paths:
+                    logger.debug(
+                        "Adding %s to sandbox as %s", path.resolve(), path.name
+                    )
+                    tf.add(path.resolve(), path.name, recursive=True)
         tar_fh.seek(0)
 
         hasher = getattr(hashlib, SANDBOX_CHECKSUM_ALGORITHM)()
@@ -89,6 +120,6 @@ async def download_sandbox(pfn: str, destination: Path, *, client: AsyncDiracCli
         fh.seek(0)
         logger.debug("Sandbox downloaded for %s", pfn)
 
-        with tarfile.open(fileobj=fh) as tf:
+        with tarfile_open(fh) as tf:
             tf.extractall(path=destination, filter="data")
         logger.debug("Extracted %s to %s", pfn, destination)

{diracx-0.0.1a46 → diracx-0.0.1a47}/diracx-cli/src/diracx/cli/internal/legacy.py

@@ -211,7 +211,6 @@ def generate_helm_values(
         },
         "rabbitmq": {"enabled": False},
         "mysql": {"enabled": False},
-        "global": {"images": {"services": "FILL ME"}, "storageClassName": "FILL ME"},
     }
 
     cfg = diraccfg.CFG().loadFromBuffer(public_cfg.read_text())
@@ -265,7 +264,7 @@ def generate_helm_values(
 
     from diracx.core.extensions import select_from_extension
 
-    for entry_point in select_from_extension(group="diracx.db.sql"):
+    for entry_point in select_from_extension(group="diracx.dbs.sql"):
         db_name = entry_point.name
         db_config = all_db_configs.get(db_name, {})
 
@@ -311,7 +310,7 @@ def generate_helm_values(
         },
     }
 
-    for entry_point in select_from_extension(group="diracx.db.os"):
+    for entry_point in select_from_extension(group="diracx.dbs.os"):
         db_name = entry_point.name
         db_config = all_db_configs.get(db_name, {})