apache-airflow-core 3.0.0__py3-none-any.whl

airflow/METRICS.md

@@ -0,0 +1,58 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+This files contains diagrams that help visualize what metrics are send, and what they represent.
+
+## DAG execution metrics
+
+```mermaid
+---
+displayMode: compact
+---
+gantt
+  title Airflow metrics for a DAG run
+  dateFormat HH:mm
+  axisFormat %H:%M
+  tickInterval 1hour
+
+  section Events
+    %% Milestones are "point in time" events, but they still take a duration.
+    %% Setting it to 2 minutes here, I don't think it has any importance.
+
+    DAG Scheduled         : milestone, dag_sched,   12:00, 2min
+    DAG Starts            : milestone, dag_start,   13:00, 2min
+    First task Scheduled  : milestone, task1_sched, 14:00, 2min
+    Task N Scheduled      : milestone, taskN_sched, 15:00, 2min
+    Task N starts running : milestone, taskN_start, 16:00, 2min
+    Task N done           : milestone, taskN_done,  17:00, 2min
+    Last task ends, DAG execution ends : milestone, end, 18:00, 2min
+
+  section Metrics
+    %% The start of the metrics can be conveniently marked with `after`,
+    %% but with this kind of diagram, we have no way to "bind" the end to a milestone,
+    %% so the durations have to be computed manually, sorry.
+    %%
+    %% If you modify the events above, make sure with your eyes that those metrics still match what they're supposed to.
+
+    dagrun.schedule_delay           : after dag_sched, 1h
+    first_task_scheduling_delay     : after dag_sched, 2h
+    duration.success/failure.dag_id : after dag_start, 5h
+    task_id.duration                : after taskN_start, 1h
+    task N landing time (only in airflow UI) : after dag_sched, 5h
+```

airflow/__init__.py

@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# We do not use "from __future__ import annotations" here because it is not supported
+# by Pycharm when we want to make sure all imports in airflow work from namespace packages
+# Adding it automatically is excluded in pyproject.toml via I002 ruff rule exclusion
+
+# Make `airflow` a namespace package, supporting installing
+# airflow.providers.* in different locations (i.e. one in site, and one in user
+# lib.)  This is required by some IDEs to resolve the import paths.
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
+
+__version__ = "3.0.0"
+
+import os
+import sys
+import warnings
+from typing import TYPE_CHECKING
+
+if os.environ.get("_AIRFLOW_PATCH_GEVENT"):
+    # If you are using gevents and start airflow webserver, you might want to run gevent monkeypatching
+    # as one of the first thing when Airflow is started. This allows gevent to patch networking and other
+    # system libraries to make them gevent-compatible before anything else patches them (for example boto)
+    from gevent.monkey import patch_all
+
+    patch_all()
+
+if sys.platform == "win32":
+    warnings.warn(
+        "Airflow currently can be run on POSIX-compliant Operating Systems. For development, "
+        "it is regularly tested on fairly modern Linux Distros and recent versions of macOS. "
+        "On Windows you can run it via WSL2 (Windows Subsystem for Linux 2) or via Linux Containers. "
+        "The work to add Windows support is tracked via https://github.com/apache/airflow/issues/10388, "
+        "but it is not a high priority.",
+        category=RuntimeWarning,
+        stacklevel=1,
+    )
+
+# The configuration module initializes and validates the conf object as a side effect the first
+# time it is imported. If it is not imported before importing the settings module, the conf
+# object will then be initted/validated as a side effect of it being imported in settings,
+# however this can cause issues since those modules are very tightly coupled and can
+# very easily cause import cycles in the conf init/validate code (since downstream code from
+# those functions likely import settings).
+# configuration is therefore initted early here, simply by importing it.
+from airflow import configuration, settings
+
+__all__ = [
+    "__version__",
+    "DAG",
+    "Asset",
+    "XComArg",
+    # TODO: Remove this module in Airflow 3.2
+    "Dataset",
+]
+
+# Perform side-effects unless someone has explicitly opted out before import
+# WARNING: DO NOT USE THIS UNLESS YOU REALLY KNOW WHAT YOU'RE DOING.
+# This environment variable prevents proper initialization, and things like
+# configs, logging, the ORM, etc. will be broken. It is only useful if you only
+# access certain trivial constants and free functions (e.g. `__version__`).
+if not os.environ.get("_AIRFLOW__AS_LIBRARY", None):
+    settings.initialize()
+
+# Things to lazy import in form {local_name: ('target_module', 'target_name', 'deprecated')}
+__lazy_imports: dict[str, tuple[str, str, bool]] = {
+    "DAG": (".models.dag", "DAG", False),
+    "Asset": (".assets", "Asset", False),
+    "XComArg": (".models.xcom_arg", "XComArg", False),
+    "version": (".version", "", False),
+    # Deprecated lazy imports
+    "AirflowException": (".exceptions", "AirflowException", True),
+    "Dataset": (".sdk.definitions.asset", "Dataset", True),
+}
+if TYPE_CHECKING:
+    # These objects are imported by PEP-562, however, static analyzers and IDE's
+    # have no idea about typing of these objects.
+    # Add it under TYPE_CHECKING block should help with it.
+    from airflow.models.dag import DAG
+    from airflow.models.xcom_arg import XComArg
+    from airflow.sdk.definitions.asset import Asset, Dataset
+
+
+def __getattr__(name: str):
+    # PEP-562: Lazy loaded attributes on python modules
+    module_path, attr_name, deprecated = __lazy_imports.get(name, ("", "", False))
+    if not module_path:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    if deprecated:
+        warnings.warn(
+            f"Import {name!r} directly from the airflow module is deprecated and "
+            f"will be removed in the future. Please import it from 'airflow{module_path}.{attr_name}'.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+    import importlib
+
+    mod = importlib.import_module(module_path, __name__)
+    if attr_name:
+        val = getattr(mod, attr_name)
+    else:
+        val = mod
+
+    # Store for next time
+    globals()[name] = val
+    return val
+
+
+if not settings.LAZY_LOAD_PROVIDERS:
+    from airflow.providers_manager import ProvidersManager
+
+    manager = ProvidersManager()
+    manager.initialize_providers_list()
+    manager.initialize_providers_hooks()
+    manager.initialize_providers_extra_links()
+if not settings.LAZY_LOAD_PLUGINS:
+    from airflow import plugins_manager
+
+    plugins_manager.ensure_plugins_loaded()

airflow/__main__.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# PYTHON_ARGCOMPLETE_OK
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Main executable module."""
+
+from __future__ import annotations
+
+import os
+
+import argcomplete
+
+# The configuration module initializes and validates the conf object as a side effect the first
+# time it is imported. If it is not imported before importing the settings module, the conf
+# object will then be initted/validated as a side effect of it being imported in settings,
+# however this can cause issues since those modules are very tightly coupled and can
+# very easily cause import cycles in the conf init/validate code (since downstream code from
+# those functions likely import settings).
+# Therefore importing configuration early (as the first airflow import) avoids
+# any possible import cycles with settings downstream.
+from airflow import configuration
+from airflow.cli import cli_parser
+
+
+def main():
+    conf = configuration.conf
+    if conf.get("core", "security") == "kerberos":
+        os.environ["KRB5CCNAME"] = conf.get("kerberos", "ccache")
+        os.environ["KRB5_KTNAME"] = conf.get("kerberos", "keytab")
+    parser = cli_parser.get_parser()
+    argcomplete.autocomplete(parser)
+    args = parser.parse_args()
+    if args.subcommand not in ["lazy_loaded", "version"]:
+        # Here we ensure that the default configuration is written if needed before running any command
+        # that might need it. This used to be done during configuration initialization but having it
+        # in main ensures that it is not done during tests and other ways airflow imports are used
+        from airflow.configuration import write_default_airflow_configuration_if_needed
+
+        conf = write_default_airflow_configuration_if_needed()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

airflow/_vendor/README.md

@@ -0,0 +1,36 @@
+# Vendor package
+
+## What vendored packages are for
+
+The `airflow._vendor` package is foreseen for vendoring in packages, that we have to modify ourselves
+because authors of the packages do not have time to modify them themselves. This is often temporary
+and once the packages implement fixes that we need, and then we remove the packages from
+the `_vendor` package.
+
+All Vendored libraries must follow these rules:
+
+1. Vendored libraries must be pure Python--no compiling (so that we do not have to release multi-platform airflow packages on PyPI).
+2. Source code for the library is included in this directory.
+3. License must be included in this repo and in the [LICENSE](../../LICENSE) file and in the
+   [licenses](../../3rd-party-licenses) folder.
+4. Requirements of the library become requirements of airflow core.
+5. Version of the library should be included in the [vendor.md](vendor.md) file.
+6. No modifications to the library may be made in the initial commit.
+7. Apply the fixes necessary to use the vendored library as separate commits - each package separately,
+   so that they can be cherry-picked later if we upgrade the vendored package. Changes to airflow code to
+   use the vendored packages should be applied as separate commits/PRs.
+8. The `_vendor` packages should be excluded from any refactoring, static checks and automated fixes.
+
+## Adding and upgrading a vendored package
+
+Way to vendor a library or update a version:
+
+1. Update ``vendor.md`` with the library, version, and SHA256 (`pypi` provides hashes as of recently)
+2. Remove all old files and directories of the old version.
+3. Replace them with new files (only replace relevant python packages:move LICENSE )
+   * move license files to [licenses](../../3rd-party-licenses) folder
+   * remove README and any other supporting files (they can be found in PyPI)
+   * make sure to add requirements to airflow's ``pyproject.toml`` with appropriate comment stating
+     why the requirements are added and when they should be removed
+4. If you replace previous version, re-apply historical fixes from the "package" folder by
+   cherry-picking them.

airflow/_vendor/vendor.md

@@ -0,0 +1,2 @@
+| Package   | Version | File                   | SHA256                                                           |
+|-----------|---------|------------------------|------------------------------------------------------------------|

airflow/alembic.ini

@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = %(here)s/migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to help/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat help/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = scheme://localhost/airflow
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

airflow/api/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/api/client/__init__.py

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""API Client that allows interacting with Airflow API."""
+
+from __future__ import annotations
+
+from airflow.api.client.local_client import Client
+
+
+def get_current_api_client() -> Client:
+    return Client()

airflow/api/client/local_client.py

@@ -0,0 +1,95 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Local client API."""
+
+from __future__ import annotations
+
+import httpx
+
+from airflow.api.common import delete_dag, trigger_dag
+from airflow.exceptions import AirflowBadRequest, PoolNotFound
+from airflow.models.pool import Pool
+from airflow.utils.types import DagRunTriggeredByType
+
+
+class Client:
+    """Local API client implementation."""
+
+    def __init__(self, auth=None, session: httpx.Client | None = None):
+        self._session: httpx.Client = session or httpx.Client()
+        if auth:
+            self._session.auth = auth
+
+    def trigger_dag(
+        self, dag_id, run_id=None, conf=None, logical_date=None, replace_microseconds=True
+    ) -> dict | None:
+        dag_run = trigger_dag.trigger_dag(
+            dag_id=dag_id,
+            triggered_by=DagRunTriggeredByType.CLI,
+            run_id=run_id,
+            conf=conf,
+            logical_date=logical_date,
+            replace_microseconds=replace_microseconds,
+        )
+        if dag_run:
+            return {
+                "conf": dag_run.conf,
+                "dag_id": dag_run.dag_id,
+                "dag_run_id": dag_run.run_id,
+                "data_interval_start": dag_run.data_interval_start,
+                "data_interval_end": dag_run.data_interval_end,
+                "end_date": dag_run.end_date,
+                "last_scheduling_decision": dag_run.last_scheduling_decision,
+                "logical_date": dag_run.logical_date,
+                "run_type": dag_run.run_type,
+                "start_date": dag_run.start_date,
+                "state": dag_run.state,
+            }
+        return dag_run
+
+    def delete_dag(self, dag_id):
+        count = delete_dag.delete_dag(dag_id)
+        return f"Removed {count} record(s)"
+
+    def get_pool(self, name):
+        pool = Pool.get_pool(pool_name=name)
+        if not pool:
+            raise PoolNotFound(f"Pool {name} not found")
+        return pool.pool, pool.slots, pool.description, pool.include_deferred
+
+    def get_pools(self):
+        return [(p.pool, p.slots, p.description, p.include_deferred) for p in Pool.get_pools()]
+
+    def create_pool(self, name, slots, description, include_deferred):
+        if not (name and name.strip()):
+            raise AirflowBadRequest("Pool name shouldn't be empty")
+        pool_name_length = Pool.pool.property.columns[0].type.length
+        if len(name) > pool_name_length:
+            raise AirflowBadRequest(f"pool name cannot be more than {pool_name_length} characters")
+        try:
+            slots = int(slots)
+        except ValueError:
+            raise AirflowBadRequest(f"Bad value for `slots`: {slots}")
+        pool = Pool.create_or_update_pool(
+            name=name, slots=slots, description=description, include_deferred=include_deferred
+        )
+        return pool.pool, pool.slots, pool.description
+
+    def delete_pool(self, name):
+        pool = Pool.delete_pool(name=name)
+        return pool.pool, pool.slots, pool.description

airflow/api/common/__init__.py

@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/api/common/airflow_health.py

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import Any
+
+from airflow.jobs.dag_processor_job_runner import DagProcessorJobRunner
+from airflow.jobs.scheduler_job_runner import SchedulerJobRunner
+from airflow.jobs.triggerer_job_runner import TriggererJobRunner
+
+HEALTHY = "healthy"
+UNHEALTHY = "unhealthy"
+
+
+def get_airflow_health() -> dict[str, Any]:
+    """Get the health for Airflow metadatabase, scheduler and triggerer."""
+    metadatabase_status = HEALTHY
+    latest_scheduler_heartbeat = None
+    latest_triggerer_heartbeat = None
+    latest_dag_processor_heartbeat = None
+
+    scheduler_status = UNHEALTHY
+    triggerer_status: str | None = UNHEALTHY
+    dag_processor_status: str | None = UNHEALTHY
+
+    try:
+        latest_scheduler_job = SchedulerJobRunner.most_recent_job()
+
+        if latest_scheduler_job:
+            latest_scheduler_heartbeat = latest_scheduler_job.latest_heartbeat.isoformat()
+            if latest_scheduler_job.is_alive():
+                scheduler_status = HEALTHY
+    except Exception:
+        metadatabase_status = UNHEALTHY
+
+    try:
+        latest_triggerer_job = TriggererJobRunner.most_recent_job()
+
+        if latest_triggerer_job:
+            latest_triggerer_heartbeat = latest_triggerer_job.latest_heartbeat.isoformat()
+            if latest_triggerer_job.is_alive():
+                triggerer_status = HEALTHY
+        else:
+            triggerer_status = None
+    except Exception:
+        metadatabase_status = UNHEALTHY
+
+    try:
+        latest_dag_processor_job = DagProcessorJobRunner.most_recent_job()
+
+        if latest_dag_processor_job:
+            latest_dag_processor_heartbeat = latest_dag_processor_job.latest_heartbeat.isoformat()
+            if latest_dag_processor_job.is_alive():
+                dag_processor_status = HEALTHY
+        else:
+            dag_processor_status = None
+    except Exception:
+        metadatabase_status = UNHEALTHY
+
+    airflow_health_status = {
+        "metadatabase": {"status": metadatabase_status},
+        "scheduler": {
+            "status": scheduler_status,
+            "latest_scheduler_heartbeat": latest_scheduler_heartbeat,
+        },
+        "triggerer": {
+            "status": triggerer_status,
+            "latest_triggerer_heartbeat": latest_triggerer_heartbeat,
+        },
+        "dag_processor": {
+            "status": dag_processor_status,
+            "latest_dag_processor_heartbeat": latest_dag_processor_heartbeat,
+        },
+    }
+
+    return airflow_health_status