apache-airflow-providers-standard 0.1.0rc1__py3-none-any.whl → 1.0.0.dev1__py3-none-any.whl

airflow/providers/standard/operators/generic_transfer.py

@@ -1,138 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-from collections.abc import Sequence
-from typing import TYPE_CHECKING
-
-from airflow.hooks.base import BaseHook
-from airflow.models import BaseOperator
-
-if TYPE_CHECKING:
-    try:
-        from airflow.sdk.definitions.context import Context
-    except ImportError:
-        # TODO: Remove once provider drops support for Airflow 2
-        from airflow.utils.context import Context
-
-
-class GenericTransfer(BaseOperator):
-    """
-    Moves data from a connection to another.
-
-    Assuming that they both provide the required methods in their respective hooks.
-    The source hook needs to expose a `get_records` method, and the destination a
-    `insert_rows` method.
-
-    This is meant to be used on small-ish datasets that fit in memory.
-
-    :param sql: SQL query to execute against the source database. (templated)
-    :param destination_table: target table. (templated)
-    :param source_conn_id: source connection. (templated)
-    :param destination_conn_id: destination connection. (templated)
-    :param preoperator: sql statement or list of statements to be
-        executed prior to loading the data. (templated)
-    :param insert_args: extra params for `insert_rows` method.
-    """
-
-    template_fields: Sequence[str] = (
-        "source_conn_id",
-        "destination_conn_id",
-        "sql",
-        "destination_table",
-        "preoperator",
-        "insert_args",
-    )
-    template_ext: Sequence[str] = (
-        ".sql",
-        ".hql",
-    )
-    template_fields_renderers = {"preoperator": "sql"}
-    ui_color = "#b0f07c"
-
-    def __init__(
-        self,
-        *,
-        sql: str,
-        destination_table: str,
-        source_conn_id: str,
-        source_hook_params: dict | None = None,
-        destination_conn_id: str,
-        destination_hook_params: dict | None = None,
-        preoperator: str | list[str] | None = None,
-        insert_args: dict | None = None,
-        **kwargs,
-    ) -> None:
-        super().__init__(**kwargs)
-        self.sql = sql
-        self.destination_table = destination_table
-        self.source_conn_id = source_conn_id
-        self.source_hook_params = source_hook_params
-        self.destination_conn_id = destination_conn_id
-        self.destination_hook_params = destination_hook_params
-        self.preoperator = preoperator
-        self.insert_args = insert_args or {}
-
-    @classmethod
-    def get_hook(cls, conn_id: str, hook_params: dict | None = None) -> BaseHook:
-        """
-        Return default hook for this connection id.
-
-        :param conn_id: connection id
-        :param hook_params: hook parameters
-        :return: default hook for this connection
-        """
-        connection = BaseHook.get_connection(conn_id)
-        return connection.get_hook(hook_params=hook_params)
-
-    def execute(self, context: Context):
-        source_hook = self.get_hook(conn_id=self.source_conn_id, hook_params=self.source_hook_params)
-        destination_hook = self.get_hook(
-            conn_id=self.destination_conn_id, hook_params=self.destination_hook_params
-        )
-
-        self.log.info("Extracting data from %s", self.source_conn_id)
-        self.log.info("Executing: \n %s", self.sql)
-        get_records = getattr(source_hook, "get_records", None)
-        if not callable(get_records):
-            raise RuntimeError(
-                f"Hook for connection {self.source_conn_id!r} "
-                f"({type(source_hook).__name__}) has no `get_records` method"
-            )
-        else:
-            results = get_records(self.sql)
-
-        if self.preoperator:
-            run = getattr(destination_hook, "run", None)
-            if not callable(run):
-                raise RuntimeError(
-                    f"Hook for connection {self.destination_conn_id!r} "
-                    f"({type(destination_hook).__name__}) has no `run` method"
-                )
-            self.log.info("Running preoperator")
-            self.log.info(self.preoperator)
-            run(self.preoperator)
-
-        insert_rows = getattr(destination_hook, "insert_rows", None)
-        if not callable(insert_rows):
-            raise RuntimeError(
-                f"Hook for connection {self.destination_conn_id!r} "
-                f"({type(destination_hook).__name__}) has no `insert_rows` method"
-            )
-        self.log.info("Inserting rows into %s", self.destination_conn_id)
-        insert_rows(table=self.destination_table, rows=results, **self.insert_args)

airflow/providers/standard/operators/latest_only.py

@@ -1,83 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Contains an operator to run downstream tasks only for the latest scheduled DagRun."""
-
-from __future__ import annotations
-
-from collections.abc import Iterable
-from typing import TYPE_CHECKING
-
-import pendulum
-
-from airflow.operators.branch import BaseBranchOperator
-
-if TYPE_CHECKING:
-    from airflow.models import DAG, DagRun
-
-    try:
-        from airflow.sdk.definitions.context import Context
-    except ImportError:
-        # TODO: Remove once provider drops support for Airflow 2
-        from airflow.utils.context import Context
-
-
-class LatestOnlyOperator(BaseBranchOperator):
-    """
-    Skip tasks that are not running during the most recent schedule interval.
-
-    If the task is run outside the latest schedule interval (i.e. external_trigger),
-    all directly downstream tasks will be skipped.
-
-    Note that downstream tasks are never skipped if the given DAG_Run is
-    marked as externally triggered.
-    """
-
-    ui_color = "#e9ffdb"  # nyanza
-
-    def choose_branch(self, context: Context) -> str | Iterable[str]:
-        # If the DAG Run is externally triggered, then return without
-        # skipping downstream tasks
-        dag_run: DagRun = context["dag_run"]  # type: ignore[assignment]
-        if dag_run.external_trigger:
-            self.log.info("Externally triggered DAG_Run: allowing execution to proceed.")
-            return list(context["task"].get_direct_relative_ids(upstream=False))
-
-        dag: DAG = context["dag"]  # type: ignore[assignment]
-        next_info = dag.next_dagrun_info(dag.get_run_data_interval(dag_run), restricted=False)
-        now = pendulum.now("UTC")
-
-        if next_info is None:
-            self.log.info("Last scheduled execution: allowing execution to proceed.")
-            return list(context["task"].get_direct_relative_ids(upstream=False))
-
-        left_window, right_window = next_info.data_interval
-        self.log.info(
-            "Checking latest only with left_window: %s right_window: %s now: %s",
-            left_window,
-            right_window,
-            now,
-        )
-
-        if not left_window < now <= right_window:
-            self.log.info("Not latest execution, skipping downstream.")
-            # we return an empty list, thus the parent BaseBranchOperator
-            # won't exclude any downstream tasks from skipping.
-            return []
-        else:
-            self.log.info("Latest, allowing execution to proceed.")
-            return list(context["task"].get_direct_relative_ids(upstream=False))