apache-airflow-providers-cncf-kubernetes 3.1.0__py3-none-any.whl → 10.10.0rc1__py3-none-any.whl

airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This file is used for documentation purposes. Example can be found at docs/executor/kubernetes.rst
+#
+# [START template_with_dags_in_volume]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: placeholder-name
+spec:
+  containers:
+    - env:
+        - name: AIRFLOW__CORE__EXECUTOR
+          value: LocalExecutor
+        # Hard Coded Airflow Envs
+        - name: AIRFLOW__CORE__FERNET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: RELEASE-NAME-fernet-key
+              key: fernet-key
+        - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
+          valueFrom:
+            secretKeyRef:
+              name: RELEASE-NAME-airflow-metadata
+              key: connection
+        - name: AIRFLOW_CONN_AIRFLOW_DB
+          valueFrom:
+            secretKeyRef:
+              name: RELEASE-NAME-airflow-metadata
+              key: connection
+      image: dummy_image
+      imagePullPolicy: IfNotPresent
+      name: base
+      volumeMounts:
+        - mountPath: "/opt/airflow/logs"
+          name: airflow-logs
+        - mountPath: /opt/airflow/dags
+          name: airflow-dags
+          readOnly: true
+        - mountPath: /opt/airflow/airflow.cfg
+          name: airflow-config
+          readOnly: true
+          subPath: airflow.cfg
+  restartPolicy: Never
+  securityContext:
+    runAsUser: 50000
+    fsGroup: 50000
+  serviceAccountName: "RELEASE-NAME-worker-serviceaccount"
+  volumes:
+    - name: airflow-dags
+      persistentVolumeClaim:
+        claimName: RELEASE-NAME-dags
+    - emptyDir: {}
+      name: airflow-logs
+    - configMap:
+        name: RELEASE-NAME-airflow-config
+      name: airflow-config
+# [END template_with_dags_in_volume]

airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This file is used for documentation purposes. Example can be found at docs/executor/kubernetes.rst
+#
+# [START git_sync_template]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dummy-name
+spec:
+  initContainers:
+    - name: git-sync
+      image: "registry.k8s.io/git-sync/git-sync:v3.6.3"
+      env:
+        - name: GIT_SYNC_BRANCH
+          value: "v2-2-stable"
+        - name: GIT_SYNC_REPO
+          value: "https://github.com/apache/airflow.git"
+        - name: GIT_SYNC_DEPTH
+          value: "1"
+        - name: GIT_SYNC_ROOT
+          value: "/git"
+        - name: GIT_SYNC_DEST
+          value: "repo"
+        - name: GIT_SYNC_ADD_USER
+          value: "true"
+        - name: GIT_SYNC_ONE_TIME
+          value: "true"
+      volumeMounts:
+        - name: airflow-dags
+          mountPath: /git
+  containers:
+    - env:
+        - name: AIRFLOW__CORE__EXECUTOR
+          value: LocalExecutor
+        # Hard Coded Airflow Envs
+        - name: AIRFLOW__CORE__FERNET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: RELEASE-NAME-fernet-key
+              key: fernet-key
+        - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
+          valueFrom:
+            secretKeyRef:
+              name: RELEASE-NAME-airflow-metadata
+              key: connection
+        - name: AIRFLOW_CONN_AIRFLOW_DB
+          valueFrom:
+            secretKeyRef:
+              name: RELEASE-NAME-airflow-metadata
+              key: connection
+      image: dummy_image
+      imagePullPolicy: IfNotPresent
+      name: base
+      volumeMounts:
+        - mountPath: "/opt/airflow/logs"
+          name: airflow-logs
+        - mountPath: /opt/airflow/dags
+          name: airflow-dags
+          subPath: repo/airflow/example_dags
+          readOnly: false
+        - mountPath: /opt/airflow/airflow.cfg
+          name: airflow-config
+          readOnly: true
+          subPath: airflow.cfg
+  restartPolicy: Never
+  securityContext:
+    runAsUser: 50000
+    fsGroup: 50000
+  serviceAccountName: "RELEASE-NAME-worker-serviceaccount"
+  volumes:
+    - name: airflow-dags
+      emptyDir: {}
+    - name: airflow-logs
+      emptyDir: {}
+    - configMap:
+        name: RELEASE-NAME-airflow-config
+      name: airflow-config
+# [END git_sync_template]

airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2

@@ -0,0 +1,51 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+from __future__ import annotations
+
+import json
+import {{ pickling_library }}
+import sys
+
+{# Check whether Airflow is available in the environment.
+ # If it is, we'll want to ensure that we integrate any macros that are being provided
+ # by plugins prior to unpickling the task context. #}
+if sys.version_info >= (3,6):
+    try:
+        from airflow.plugins_manager import integrate_macros_plugins
+        integrate_macros_plugins()
+    except ImportError:
+        {# Airflow is not available in this environment, therefore we won't
+         # be able to integrate any plugin macros. #}
+        pass
+
+{% if op_args or op_kwargs %}
+with open(sys.argv[1], "rb") as file:
+    arg_dict = {{ pickling_library }}.load(file)
+{% else %}
+arg_dict = {"args": [], "kwargs": {}}
+{% endif %}
+
+# Script
+{{ python_callable_source }}
+res = {{ python_callable }}(*arg_dict["args"], **arg_dict["kwargs"])
+
+# Write output
+with open(sys.argv[2], "w") as file:
+    if res is not None:
+        file.write(json.dumps(res))

airflow/providers/cncf/kubernetes/python_kubernetes_script.py

@@ -0,0 +1,92 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Utilities for using the kubernetes decorator."""
+
+from __future__ import annotations
+
+import os
+from collections import deque
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined, select_autoescape
+from jinja2.nativetypes import NativeEnvironment
+
+
+def _balance_parens(after_decorator):
+    num_paren = 1
+    after_decorator = deque(after_decorator)
+    after_decorator.popleft()
+    while num_paren:
+        current = after_decorator.popleft()
+        if current == "(":
+            num_paren += 1
+        elif current == ")":
+            num_paren -= 1
+    return "".join(after_decorator)
+
+
+def remove_task_decorator(python_source: str, task_decorator_name: str) -> str:
+    """
+    Remove @task.kubernetes or similar as well as @setup and @teardown.
+
+    :param python_source: python source code
+    :param task_decorator_name: the task decorator name
+    """
+
+    def _remove_task_decorator(py_source, decorator_name):
+        if decorator_name not in py_source:
+            return python_source
+        split = python_source.split(decorator_name)
+        before_decorator, after_decorator = split[0], split[1]
+        if after_decorator[0] == "(":
+            after_decorator = _balance_parens(after_decorator)
+        if after_decorator[0] == "\n":
+            after_decorator = after_decorator[1:]
+        return before_decorator + after_decorator
+
+    decorators = ["@setup", "@teardown", task_decorator_name]
+    for decorator in decorators:
+        python_source = _remove_task_decorator(python_source, decorator)
+    return python_source
+
+
+def write_python_script(
+    jinja_context: dict,
+    filename: str,
+    render_template_as_native_obj: bool = False,
+):
+    """
+    Render the python script to a file to execute in the virtual environment.
+
+    :param jinja_context: The jinja context variables to unpack and replace with its placeholders in the
+        template file.
+    :param filename: The name of the file to dump the rendered script to.
+    :param render_template_as_native_obj: If ``True``, rendered Jinja template would be converted
+        to a native Python object
+    """
+    template_loader = FileSystemLoader(searchpath=os.path.dirname(__file__))
+    template_env: Environment
+    if render_template_as_native_obj:
+        template_env = NativeEnvironment(loader=template_loader, undefined=StrictUndefined)
+    else:
+        template_env = Environment(
+            loader=template_loader,
+            undefined=StrictUndefined,
+            autoescape=select_autoescape(["html", "xml"]),
+        )
+    template = template_env.get_template("python_kubernetes_script.jinja2")
+    template.stream(**jinja_context).dump(filename)

airflow/providers/cncf/kubernetes/resource_convert/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/cncf/kubernetes/resource_convert/configmap.py

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from kubernetes.client import models as k8s
+
+
+def convert_configmap(configmap_name) -> k8s.V1EnvFromSource:
+    """
+    Convert a str into an k8s object.
+
+    :param configmap_name: config map name
+    :return:
+    """
+    return k8s.V1EnvFromSource(config_map_ref=k8s.V1ConfigMapEnvSource(name=configmap_name))
+
+
+def convert_configmap_to_volume(
+    configmap_info: dict[str, str],
+) -> tuple[list[k8s.V1Volume], list[k8s.V1VolumeMount]]:
+    """
+    Convert a dictionary of config_map_name and mount_path into k8s volume mount object and k8s volume.
+
+    :param configmap_info: a dictionary of {config_map_name: mount_path}
+    :return:
+    """
+    volume_mounts = []
+    volumes = []
+    for config_name, mount_path in configmap_info.items():
+        volume_mounts.append(k8s.V1VolumeMount(mount_path=mount_path, name=config_name))
+        volumes.append(
+            k8s.V1Volume(
+                name=config_name,
+                config_map=k8s.V1ConfigMapVolumeSource(name=config_name),
+            )
+        )
+
+    return volumes, volume_mounts

airflow/providers/cncf/kubernetes/resource_convert/env_variable.py

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from kubernetes.client import models as k8s
+
+from airflow.exceptions import AirflowException
+
+
+def convert_env_vars(env_vars) -> list[k8s.V1EnvVar]:
+    """
+    Convert a dictionary of key:value into a list of env_vars.
+
+    :param env_vars:
+    :return:
+    """
+    if isinstance(env_vars, dict):
+        res = []
+        for k, v in env_vars.items():
+            res.append(k8s.V1EnvVar(name=k, value=v))
+        return res
+    if isinstance(env_vars, list):
+        if all([isinstance(e, k8s.V1EnvVar) for e in env_vars]):
+            return env_vars
+    raise AirflowException(f"Expected dict or list of V1EnvVar, got {type(env_vars)}")

airflow/providers/cncf/kubernetes/resource_convert/secret.py

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from kubernetes.client import models as k8s
+
+
+def convert_secret(secret_name: str) -> k8s.V1EnvFromSource:
+    """
+    Convert a str into an k8s object.
+
+    :param secret_name:
+    :return:
+    """
+    return k8s.V1EnvFromSource(secret_ref=k8s.V1SecretEnvSource(name=secret_name))
+
+
+def convert_image_pull_secrets(image_pull_secrets: str) -> list[k8s.V1LocalObjectReference]:
+    """
+    Convert an image pull secret name into k8s local object reference.
+
+    :param image_pull_secrets: comma separated string that contains secrets
+    :return:
+    """
+    secrets = image_pull_secrets.split(",")
+    return [k8s.V1LocalObjectReference(name=secret) for secret in secrets]

airflow/providers/cncf/kubernetes/secret.py

@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Classes for interacting with Kubernetes API."""
+
+from __future__ import annotations
+
+import copy
+import uuid
+
+from kubernetes.client import models as k8s
+
+from airflow.exceptions import AirflowConfigException
+from airflow.providers.cncf.kubernetes.k8s_model import K8SModel
+
+
+class Secret(K8SModel):
+    """Defines Kubernetes Secret Volume."""
+
+    def __init__(self, deploy_type, deploy_target, secret, key=None, items=None):
+        """
+        Initialize a Kubernetes Secret Object.
+
+        Used to track requested secrets from the user.
+
+        :param deploy_type: The type of secret deploy in Kubernetes, either `env` or
+            `volume`
+        :param deploy_target: (Optional) The environment variable when
+            `deploy_type` `env` or file path when `deploy_type` `volume` where
+            expose secret. If `key` is not provided deploy target should be None.
+        :param secret: Name of the secrets object in Kubernetes
+        :param key: (Optional) Key of the secret within the Kubernetes Secret
+            if not provided in `deploy_type` `env` it will mount all secrets in object
+        :param items: (Optional) items that can be added to a volume secret for specifying projects of
+        secret keys to paths
+        https://kubernetes.io/docs/concepts/configuration/secret/#projection-of-secret-keys-to-specific-paths
+        """
+        if deploy_type not in ("env", "volume"):
+            raise AirflowConfigException("deploy_type must be env or volume")
+
+        self.deploy_type = deploy_type
+        self.deploy_target = deploy_target
+        self.items = items or []
+
+        if deploy_target is not None and deploy_type == "env":
+            # if deploying to env, capitalize the deploy target
+            self.deploy_target = deploy_target.upper()
+
+        if key is not None and deploy_target is None:
+            raise AirflowConfigException("If `key` is set, `deploy_target` should not be None")
+
+        self.secret = secret
+        self.key = key
+
+    def to_env_secret(self) -> k8s.V1EnvVar:
+        """Store es environment secret."""
+        return k8s.V1EnvVar(
+            name=self.deploy_target,
+            value_from=k8s.V1EnvVarSource(
+                secret_key_ref=k8s.V1SecretKeySelector(name=self.secret, key=self.key)
+            ),
+        )
+
+    def to_env_from_secret(self) -> k8s.V1EnvFromSource:
+        """Read from environment to secret."""
+        return k8s.V1EnvFromSource(secret_ref=k8s.V1SecretEnvSource(name=self.secret))
+
+    def to_volume_secret(self) -> tuple[k8s.V1Volume, k8s.V1VolumeMount]:
+        """Convert to volume secret."""
+        vol_id = f"secretvol{uuid.uuid4()}"
+        volume = k8s.V1Volume(name=vol_id, secret=k8s.V1SecretVolumeSource(secret_name=self.secret))
+        if self.items:
+            volume.secret.items = self.items
+        return (volume, k8s.V1VolumeMount(mount_path=self.deploy_target, name=vol_id, read_only=True))
+
+    def attach_to_pod(self, pod: k8s.V1Pod) -> k8s.V1Pod:
+        """Attach to pod."""
+        cp_pod = copy.deepcopy(pod)
+
+        if self.deploy_type == "volume":
+            volume, volume_mount = self.to_volume_secret()
+            if cp_pod.spec.volumes is None:
+                cp_pod.spec.volumes = []
+            cp_pod.spec.volumes.append(volume)
+            if cp_pod.spec.containers[0].volume_mounts is None:
+                cp_pod.spec.containers[0].volume_mounts = []
+            cp_pod.spec.containers[0].volume_mounts.append(volume_mount)
+
+        if self.deploy_type == "env" and self.key is not None:
+            env = self.to_env_secret()
+            if cp_pod.spec.containers[0].env is None:
+                cp_pod.spec.containers[0].env = []
+            cp_pod.spec.containers[0].env.append(env)
+
+        if self.deploy_type == "env" and self.key is None:
+            env_from = self.to_env_from_secret()
+            if cp_pod.spec.containers[0].env_from is None:
+                cp_pod.spec.containers[0].env_from = []
+            cp_pod.spec.containers[0].env_from.append(env_from)
+
+        return cp_pod
+
+    def __eq__(self, other):
+        return (
+            self.deploy_type == other.deploy_type
+            and self.deploy_target == other.deploy_target
+            and self.secret == other.secret
+            and self.key == other.key
+        )
+
+    def __hash__(self):
+        return hash((self.deploy_type, self.deploy_target, self.secret, self.key))
+
+    def __repr__(self):
+        return f"Secret({self.deploy_type}, {self.deploy_target}, {self.secret}, {self.key})"