vigie-spark-operator 0.1.2__tar.gz

vigie_spark_operator-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 HEMA Tiemoko Thierry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vigie_spark_operator-0.1.2/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: vigie-spark-operator
+Version: 0.1.2
+Summary: Airflow operator that submits Kubeflow SparkApplications on Kubernetes (AF2 + AF3)
+Author: HEMA Tiemoko Thierry
+License: MIT
+Project-URL: Homepage, https://github.com/Jaali-co/vigie-spark-operator
+Project-URL: Changelog, https://github.com/Jaali-co/vigie-spark-operator/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/Jaali-co/vigie-spark-operator/issues
+Keywords: airflow,spark,kubernetes,spark-operator
+Classifier: Development Status :: 4 - Beta
+Classifier: Framework :: Apache Airflow
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: kubernetes>=28.1.0
+Provides-Extra: airflow
+Requires-Dist: apache-airflow<4,>=2.7; extra == "airflow"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: pytest-cov>=4.1; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=5.0; extra == "dev"
+Dynamic: license-file
+
+# vigie-spark-operator
+
+Open-source Airflow operator that submits a Kubeflow **SparkApplication**
+(Spark Operator on Kubernetes). Works with **Airflow 2.x and 3.x**.
+
+Maintainer: **HEMA Tiemoko Thierry** · License: **MIT**
+
+## Requirements
+
+| | |
+|---|---|
+| Python | `>=3.9` |
+| Runtime dep | `kubernetes>=28.1.0` |
+| Airflow (runtime) | `>=2.7,<4` — provided by the worker image, or install extra `[airflow]` |
+| Cluster | [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator) CRD `SparkApplication` installed; RBAC for the Airflow worker SA |
+
+Declared in `pyproject.toml`. Dev tools: see `requirements-dev.txt`.
+
+```bash
+pip install vigie-spark-operator
+# or with Airflow pinned as dependency (usually not needed inside an AF image)
+pip install "vigie-spark-operator[airflow]"
+```
+
+## Quick start
+
+```python
+from vigie_spark_operator import SparkK8sOperator
+
+SparkK8sOperator(
+    task_id="spark_pi",
+    name="pi",
+    namespace="spark-jobs",
+    image="spark:3.5.1",
+    application_type="Scala",
+    main_class="org.apache.spark.examples.SparkPi",
+    main_application_file="local:///opt/spark/examples/jars/spark-examples_2.12-3.5.1.jar",
+    arguments=["10"],
+    spark_version="3.5.1",
+    service_account="spark",
+    dry_run=False,
+)
+```
+
+### Optional Vigie capacity labels
+
+Generic by default (no `managed-by`). For Vigie Supervision, pass
+`managed_by="vigie"` so driver/executor get `managed-by=vigie` plus
+`dag_id` / `task_id` / `run_id`:
+
+```python
+SparkK8sOperator(..., managed_by="vigie")
+```
+
+Disable context labels entirely: `inject_airflow_labels=False`.
+
+## Install on Airflow workers
+
+Preferred: bake the wheel into the worker image.
+
+Alternative: `_PIP_ADDITIONAL_REQUIREMENTS=vigie-spark-operator==0.1.2` (slower, less reproducible).
+
+AF2 / AF3: the package selects `airflow.models` (AF2) or `airflow.sdk` (AF3) via `compat.py`.
+
+## Main parameters
+
+| Parameter | Default | Notes |
+|-----------|---------|--------|
+| `name` | *(required)* | Base name for the SparkApplication |
+| `namespace` | *(required)* | Target K8s namespace |
+| `image` | *(required)* | Spark runtime image |
+| `main_application_file` | *(required)* | App entry (`local://…`, `s3a://…`, …) |
+| `application_type` | `"Python"` | `Python` / `Scala` / `Java` / `R` |
+| `main_class` | `None` | Required for Scala/Java |
+| `arguments` | `[]` | App args |
+| `spark_version` | `"3.1.1"` | Reported to the Spark Operator |
+| `driver_cores` / `driver_memory` | `1` / `"512m"` | Driver resources |
+| `driver_cores_request` | `"20m"` | K8s CPU request |
+| `executor_cores` / `executor_memory` | `1` / `"512m"` | Executor resources |
+| `executor_cores_request` | `"400m"` | K8s CPU request |
+| `executor_instances` | `2` | Number of executors |
+| `service_account` | `"spark"` | Pod SA in the job namespace |
+| `spark_conf` | `{}` | Extra Spark conf |
+| `labels` / `env` / `jars` | `{}` / `{}` / `[]` | Merged labels, env vars, extra jars |
+| `pvc_configs` / `emptydir_configs` | `[]` | Volume mounts (`PvcConfig` / `EmptyDirConfig` or dicts) |
+| `run_as_user` / `fs_group` | `0` / `0` | Shared security context |
+| `allow_privilege_escalation` | `False` | Shared; optional `driver_*` / `executor_*` overrides |
+| `timeout_job` | `3600` | Job timeout (s); graceful delete then force-kill |
+| `timeout` | `None` | Legacy alias → sets `timeout_job` |
+| `poll_interval` | `10` | Status poll (s) |
+| `api_error_retries` | `5` | Retries on K8s `429` / `503` |
+| `in_cluster` | `True` | Use in-cluster kubeconfig |
+| `fail_on_unschedulable` | `True` | Fail if pods stay Pending |
+| `dry_run` | `False` | Build + log manifest, no submit |
+| `inject_airflow_labels` | `True` | Inject `dag_id` / `task_id` / `run_id` |
+| `managed_by` | `None` | e.g. `"vigie"` for capacity collector |
+
+Also accepts standard `BaseOperator` kwargs (`task_id`, `queue`, `executor`, …).
+
+## Behaviour highlights
+
+- Labels on **metadata, driver and executor**
+- `dry_run=True`: validates resources summary, exits 0 without creating the CR
+- Exponential backoff on transient K8s API errors (`429`, `503`)
+- Near `timeout_job`: delete SparkApplication, then force-kill leftover pods
+
+## Development
+
+```bash
+python3 -m venv .venv && source .venv/bin/activate
+pip install -r requirements-dev.txt
+pip install -e .
+pytest -q
+# packaging smoke (build → twine check → clean install)
+../scripts/test_pypi_packaging.sh .
+```
+
+## Links
+
+- Changelog: [CHANGELOG.md](CHANGELOG.md)
+- Issues: https://github.com/Jaali-co/vigie-spark-operator/issues

vigie_spark_operator-0.1.2/README.md

@@ -0,0 +1,121 @@
+# vigie-spark-operator
+
+Open-source Airflow operator that submits a Kubeflow **SparkApplication**
+(Spark Operator on Kubernetes). Works with **Airflow 2.x and 3.x**.
+
+Maintainer: **HEMA Tiemoko Thierry** · License: **MIT**
+
+## Requirements
+
+| | |
+|---|---|
+| Python | `>=3.9` |
+| Runtime dep | `kubernetes>=28.1.0` |
+| Airflow (runtime) | `>=2.7,<4` — provided by the worker image, or install extra `[airflow]` |
+| Cluster | [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator) CRD `SparkApplication` installed; RBAC for the Airflow worker SA |
+
+Declared in `pyproject.toml`. Dev tools: see `requirements-dev.txt`.
+
+```bash
+pip install vigie-spark-operator
+# or with Airflow pinned as dependency (usually not needed inside an AF image)
+pip install "vigie-spark-operator[airflow]"
+```
+
+## Quick start
+
+```python
+from vigie_spark_operator import SparkK8sOperator
+
+SparkK8sOperator(
+    task_id="spark_pi",
+    name="pi",
+    namespace="spark-jobs",
+    image="spark:3.5.1",
+    application_type="Scala",
+    main_class="org.apache.spark.examples.SparkPi",
+    main_application_file="local:///opt/spark/examples/jars/spark-examples_2.12-3.5.1.jar",
+    arguments=["10"],
+    spark_version="3.5.1",
+    service_account="spark",
+    dry_run=False,
+)
+```
+
+### Optional Vigie capacity labels
+
+Generic by default (no `managed-by`). For Vigie Supervision, pass
+`managed_by="vigie"` so driver/executor get `managed-by=vigie` plus
+`dag_id` / `task_id` / `run_id`:
+
+```python
+SparkK8sOperator(..., managed_by="vigie")
+```
+
+Disable context labels entirely: `inject_airflow_labels=False`.
+
+## Install on Airflow workers
+
+Preferred: bake the wheel into the worker image.
+
+Alternative: `_PIP_ADDITIONAL_REQUIREMENTS=vigie-spark-operator==0.1.2` (slower, less reproducible).
+
+AF2 / AF3: the package selects `airflow.models` (AF2) or `airflow.sdk` (AF3) via `compat.py`.
+
+## Main parameters
+
+| Parameter | Default | Notes |
+|-----------|---------|--------|
+| `name` | *(required)* | Base name for the SparkApplication |
+| `namespace` | *(required)* | Target K8s namespace |
+| `image` | *(required)* | Spark runtime image |
+| `main_application_file` | *(required)* | App entry (`local://…`, `s3a://…`, …) |
+| `application_type` | `"Python"` | `Python` / `Scala` / `Java` / `R` |
+| `main_class` | `None` | Required for Scala/Java |
+| `arguments` | `[]` | App args |
+| `spark_version` | `"3.1.1"` | Reported to the Spark Operator |
+| `driver_cores` / `driver_memory` | `1` / `"512m"` | Driver resources |
+| `driver_cores_request` | `"20m"` | K8s CPU request |
+| `executor_cores` / `executor_memory` | `1` / `"512m"` | Executor resources |
+| `executor_cores_request` | `"400m"` | K8s CPU request |
+| `executor_instances` | `2` | Number of executors |
+| `service_account` | `"spark"` | Pod SA in the job namespace |
+| `spark_conf` | `{}` | Extra Spark conf |
+| `labels` / `env` / `jars` | `{}` / `{}` / `[]` | Merged labels, env vars, extra jars |
+| `pvc_configs` / `emptydir_configs` | `[]` | Volume mounts (`PvcConfig` / `EmptyDirConfig` or dicts) |
+| `run_as_user` / `fs_group` | `0` / `0` | Shared security context |
+| `allow_privilege_escalation` | `False` | Shared; optional `driver_*` / `executor_*` overrides |
+| `timeout_job` | `3600` | Job timeout (s); graceful delete then force-kill |
+| `timeout` | `None` | Legacy alias → sets `timeout_job` |
+| `poll_interval` | `10` | Status poll (s) |
+| `api_error_retries` | `5` | Retries on K8s `429` / `503` |
+| `in_cluster` | `True` | Use in-cluster kubeconfig |
+| `fail_on_unschedulable` | `True` | Fail if pods stay Pending |
+| `dry_run` | `False` | Build + log manifest, no submit |
+| `inject_airflow_labels` | `True` | Inject `dag_id` / `task_id` / `run_id` |
+| `managed_by` | `None` | e.g. `"vigie"` for capacity collector |
+
+Also accepts standard `BaseOperator` kwargs (`task_id`, `queue`, `executor`, …).
+
+## Behaviour highlights
+
+- Labels on **metadata, driver and executor**
+- `dry_run=True`: validates resources summary, exits 0 without creating the CR
+- Exponential backoff on transient K8s API errors (`429`, `503`)
+- Near `timeout_job`: delete SparkApplication, then force-kill leftover pods
+
+## Development
+
+```bash
+python3 -m venv .venv && source .venv/bin/activate
+pip install -r requirements-dev.txt
+pip install -e .
+pytest -q
+# packaging smoke (build → twine check → clean install)
+../scripts/test_pypi_packaging.sh .
+```
+
+## Links
+
+- Changelog: [CHANGELOG.md](CHANGELOG.md)
+- Issues: https://github.com/Jaali-co/vigie-spark-operator/issues

vigie_spark_operator-0.1.2/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vigie-spark-operator"
+version = "0.1.2"
+description = "Airflow operator that submits Kubeflow SparkApplications on Kubernetes (AF2 + AF3)"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "HEMA Tiemoko Thierry" }]
+keywords = ["airflow", "spark", "kubernetes", "spark-operator"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Framework :: Apache Airflow",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Topic :: System :: Distributed Computing",
+]
+dependencies = [
+  "kubernetes>=28.1.0",
+]
+
+[project.optional-dependencies]
+airflow = ["apache-airflow>=2.7,<4"]
+dev = ["pytest>=7.4", "pytest-cov>=4.1", "build>=1.2", "twine>=5.0"]
+
+[project.urls]
+Homepage = "https://github.com/Jaali-co/vigie-spark-operator"
+Changelog = "https://github.com/Jaali-co/vigie-spark-operator/blob/main/CHANGELOG.md"
+Issues = "https://github.com/Jaali-co/vigie-spark-operator/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["vigie_spark_operator*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["."]

vigie_spark_operator-0.1.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

vigie_spark_operator-0.1.2/tests/test_compat_af2_af3.py

@@ -0,0 +1,90 @@
+"""Tests for AF2/AF3 BaseOperator resolution."""
+
+from __future__ import annotations
+
+import importlib
+import sys
+import types
+from unittest.mock import MagicMock
+
+
+def _install_af2_stub():
+    airflow = types.ModuleType("airflow")
+    airflow.__version__ = "2.10.5"
+    models = types.ModuleType("airflow.models")
+
+    class BaseOperator:
+        template_fields = ()
+
+        def __init__(self, **kwargs):
+            self.log = MagicMock()
+
+    class Variable:
+        @staticmethod
+        def get(*a, **k):
+            return {}
+
+    models.BaseOperator = BaseOperator
+    models.Variable = Variable
+    sys.modules["airflow"] = airflow
+    sys.modules["airflow.models"] = models
+    sys.modules.pop("airflow.sdk", None)
+    sys.modules.pop("vigie_spark_operator.compat", None)
+
+
+def _install_af3_stub():
+    airflow = types.ModuleType("airflow")
+    airflow.__version__ = "3.2.2"
+    sdk = types.ModuleType("airflow.sdk")
+
+    class BaseOperator:
+        template_fields = ()
+
+        def __init__(self, **kwargs):
+            self.log = MagicMock()
+
+    class Variable:
+        @staticmethod
+        def get(*a, **k):
+            return {}
+
+    sdk.BaseOperator = BaseOperator
+    sdk.Variable = Variable
+    models = types.ModuleType("airflow.models")
+    models.BaseOperator = object  # must not be used on AF3 path
+    sys.modules["airflow"] = airflow
+    sys.modules["airflow.sdk"] = sdk
+    sys.modules["airflow.models"] = models
+    sys.modules.pop("vigie_spark_operator.compat", None)
+
+
+def test_compat_af2_uses_models():
+    _install_af2_stub()
+    compat = importlib.import_module("vigie_spark_operator.compat")
+    importlib.reload(compat)
+    assert compat.AIRFLOW_V_3_0_PLUS is False
+    assert compat.BaseOperator is sys.modules["airflow.models"].BaseOperator
+
+
+def test_compat_af3_uses_sdk():
+    _install_af3_stub()
+    compat = importlib.import_module("vigie_spark_operator.compat")
+    importlib.reload(compat)
+    assert compat.AIRFLOW_V_3_0_PLUS is True
+    assert compat.BaseOperator is sys.modules["airflow.sdk"].BaseOperator
+    assert compat.Variable is sys.modules["airflow.sdk"].Variable
+
+
+def test_compat_af2_variable_lazy():
+    _install_af2_stub()
+    models = sys.modules["airflow.models"]
+
+    class Variable:
+        @staticmethod
+        def get(*a, **k):
+            return "x"
+
+    models.Variable = Variable
+    compat = importlib.import_module("vigie_spark_operator.compat")
+    importlib.reload(compat)
+    assert compat.Variable is Variable

vigie_spark_operator-0.1.2/tests/test_operator.py

@@ -0,0 +1,219 @@
+"""Tests unitaires — labels et dry_run SparkK8sOperator."""
+
+from __future__ import annotations
+
+import sys
+import types
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+# Stub Airflow avant import de l'operator (pas besoin d'Airflow installé pour ces tests)
+_airflow = types.ModuleType("airflow")
+_models = types.ModuleType("airflow.models")
+
+
+class _FakeBaseOperator:
+    template_fields: tuple = ()
+
+    def __init__(self, **kwargs):
+        self.log = MagicMock()
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+
+
+_models.BaseOperator = _FakeBaseOperator
+
+
+class _FakeVariable:
+    @staticmethod
+    def get(*_a, **_k):
+        return {}
+
+
+_models.Variable = _FakeVariable
+sys.modules["airflow"] = _airflow
+sys.modules["airflow.models"] = _models
+
+# Stub kubernetes minimal (chargé au import operator)
+_k8s = types.ModuleType("kubernetes")
+_k8s_client = types.ModuleType("kubernetes.client")
+_k8s_config = types.ModuleType("kubernetes.config")
+_k8s_exc = types.ModuleType("kubernetes.client.exceptions")
+
+
+class _ApiException(Exception):
+    def __init__(self, status=500, reason="err"):
+        self.status = status
+        self.reason = reason
+
+
+_k8s_exc.ApiException = _ApiException
+_k8s_client.exceptions = _k8s_exc
+_k8s_client.CustomObjectsApi = MagicMock
+_k8s_client.CoreV1Api = MagicMock
+_k8s_client.V1DeleteOptions = MagicMock
+_k8s.client = _k8s_client
+_k8s.config = _k8s_config
+sys.modules["kubernetes"] = _k8s
+sys.modules["kubernetes.client"] = _k8s_client
+sys.modules["kubernetes.config"] = _k8s_config
+sys.modules["kubernetes.client.exceptions"] = _k8s_exc
+
+# Force re-bind BaseOperator from our AF stub (compat may have been imported earlier).
+for mod in (
+    "vigie_spark_operator.compat",
+    "vigie_spark_operator.operator",
+    "vigie_spark_operator",
+):
+    sys.modules.pop(mod, None)
+
+from vigie_spark_operator import SparkK8sOperator, __version__
+from vigie_spark_operator.labels import airflow_labels, sanitize_label_value
+from vigie_spark_operator.retry import k8s_call_with_retry
+
+
+def _ctx(dag_id="my-dag", task_id="spark_pi", run_id="manual__2026-09-05T10:00:00+00:00"):
+    return {
+        "dag": SimpleNamespace(dag_id=dag_id),
+        "task": SimpleNamespace(task_id=task_id),
+        "run_id": run_id,
+        "ts_nodash": "20260905T100000",
+        "ti": SimpleNamespace(try_number=1),
+    }
+
+
+def test_version():
+    assert __version__ == "0.1.2"
+
+
+def test_sanitize_truncates_and_cleans():
+    long_run = "x" * 80
+    assert len(sanitize_label_value(long_run)) <= 63
+    assert sanitize_label_value("dag/with spaces!") == "dag-with-spaces"
+
+
+def test_airflow_labels_from_context():
+    labels = airflow_labels(_ctx())
+    assert labels["dag_id"] == "my-dag"
+    assert labels["task_id"] == "spark_pi"
+    assert "managed-by" not in labels
+    assert len(labels["run_id"]) <= 63
+
+
+def test_airflow_labels_managed_by():
+    labels = airflow_labels(_ctx(), managed_by="vigie")
+    assert labels["managed-by"] == "vigie"
+
+
+def test_security_defaults():
+    op = SparkK8sOperator(
+        task_id="spark_pi",
+        name="pi",
+        namespace="ns",
+        image="spark:3.5.1",
+        main_application_file="local:///x.py",
+    )
+    spec = op._build_spark_app_spec(_ctx())
+    for side in ("driver", "executor"):
+        assert spec["spec"][side]["securityContext"]["runAsUser"] == 0
+        assert spec["spec"][side]["podSecurityContext"]["fsGroup"] == 0
+        assert spec["spec"][side]["securityContext"]["allowPrivilegeEscalation"] is False
+
+
+def test_security_overrides_shared_and_per_role():
+    op = SparkK8sOperator(
+        task_id="spark_pi",
+        name="pi",
+        namespace="ns",
+        image="spark:3.5.1",
+        main_application_file="local:///x.py",
+        run_as_user=185,
+        fs_group=185,
+        executor_run_as_user=1000,
+    )
+    spec = op._build_spark_app_spec(_ctx())
+    assert spec["spec"]["driver"]["securityContext"]["runAsUser"] == 185
+    assert spec["spec"]["driver"]["podSecurityContext"]["fsGroup"] == 185
+    assert spec["spec"]["executor"]["securityContext"]["runAsUser"] == 1000
+    assert spec["spec"]["executor"]["podSecurityContext"]["fsGroup"] == 185
+
+
+def test_build_spec_injects_labels_on_driver_and_executor():
+    op = SparkK8sOperator(
+        task_id="spark_pi",
+        name="pi",
+        namespace="airflow2-spark-jobs",
+        image="spark:3.5.1",
+        main_application_file="local:///opt/spark/examples/jars/x.jar",
+        application_type="Scala",
+        main_class="org.apache.spark.examples.SparkPi",
+        executor_instances=1,
+        managed_by="vigie",
+    )
+    spec = op._build_spark_app_spec(_ctx())
+    for side in ("driver", "executor"):
+        labels = spec["spec"][side]["labels"]
+        assert labels["dag_id"] == "my-dag"
+        assert labels["task_id"] == "spark_pi"
+        assert labels["managed-by"] == "vigie"
+        assert "run_id" in labels
+    assert spec["metadata"]["labels"]["dag_id"] == "my-dag"
+
+
+def test_build_spec_without_managed_by_omits_label():
+    op = SparkK8sOperator(
+        task_id="spark_pi",
+        name="pi",
+        namespace="ns",
+        image="spark:3.5.1",
+        main_application_file="local:///x.py",
+    )
+    labels = op._build_spark_app_spec(_ctx())["spec"]["driver"]["labels"]
+    assert "managed-by" not in labels
+    assert labels["dag_id"] == "my-dag"
+
+
+def test_dry_run_returns_summary_without_k8s():
+    op = SparkK8sOperator(
+        task_id="spark_pi",
+        name="pi",
+        namespace="airflow2-spark-jobs",
+        image="spark:3.5.1",
+        main_application_file="local:///x.py",
+        application_type="Python",
+        dry_run=True,
+        executor_instances=2,
+        driver_memory="1g",
+        executor_memory="2g",
+    )
+    # dry_run ne doit pas appeler _load_kube
+    op._load_kube = MagicMock(side_effect=AssertionError("k8s should not load"))
+    summary = op.execute(_ctx())
+    assert summary["driver"]["memory"] == "1g"
+    assert summary["executor"]["instances"] == 2
+    assert summary["labels"]["driver"]["dag_id"] == "my-dag"
+    op._load_kube.assert_not_called()
+
+
+def test_k8s_retry_on_429():
+    calls = {"n": 0}
+
+    def flaky():
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise _ApiException(status=429, reason="too many")
+        return "ok"
+
+    assert k8s_call_with_retry(flaky, max_retries=3, base_delay=0.01) == "ok"
+    assert calls["n"] == 3
+
+
+def test_k8s_retry_gives_up_on_403():
+    def forbidden():
+        raise _ApiException(status=403, reason="nope")
+
+    with pytest.raises(_ApiException) as ei:
+        k8s_call_with_retry(forbidden, max_retries=3, base_delay=0.01)
+    assert ei.value.status == 403

vigie_spark_operator-0.1.2/vigie_spark_operator/__init__.py

@@ -0,0 +1,16 @@
+"""vigie_spark_operator — SparkK8sOperator (Airflow → SparkApplication K8s)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+__version__ = "0.1.2"
+__all__ = ["SparkK8sOperator", "__version__"]
+
+
+def __getattr__(name: str) -> Any:
+    if name == "SparkK8sOperator":
+        from vigie_spark_operator.operator import SparkK8sOperator
+
+        return SparkK8sOperator
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")