flowtask 5.12.18__tar.gz → 5.13.0__tar.gz

{flowtask-5.12.18 → flowtask-5.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.12.18
+Version: 5.13.0
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -109,8 +109,9 @@ Requires-Dist: querysource>=3.17.0
 Requires-Dist: asyncdb[all]>=2.11.7
 Requires-Dist: navconfig[default,uvloop]>=1.7.13
 Requires-Dist: navigator-auth>=0.15.8
+Requires-Dist: aiohttp>=3.14
 Requires-Dist: navigator-api[locale,uvloop]>=2.12.22
-Requires-Dist: ai-parrot[llms]>=0.17.4
+Requires-Dist: ai-parrot[llms]>=1.0.5
 Requires-Dist: msal<2,>=1.30.0
 Requires-Dist: azure-core<2,>=1.38.0
 Requires-Dist: azure-identity<2,>=1.20.0
@@ -132,12 +133,12 @@ Requires-Dist: gitpython>=3.1.50; extra == "git"
 Provides-Extra: scoring
 Requires-Dist: h3>=4.0; extra == "scoring"
 Provides-Extra: ai
-Requires-Dist: ai-parrot[agents,graphindex,images,llms,retrieval]>=0.20.4; extra == "ai"
-Requires-Dist: ai-parrot-loaders[audio,document,ebook,ml,pdf,scraping,video,web,youtube]>=0.1.0; extra == "ai"
-Requires-Dist: ai-parrot-tools[analysis,arxiv,aws,codeinterpreter,db,docker,excel,finance,flowtask,git,google,jira,messaging,pdf,pulumi,sandbox,scraping,sitesearch,slack,weather,wikipedia]>=0.1.0; extra == "ai"
-Requires-Dist: ai-parrot-pipelines>=0.1.0; extra == "ai"
-Requires-Dist: ai-parrot-server[all]>=0.1.0; extra == "ai"
-Requires-Dist: ai-parrot-integrations[all]>=0.0.17; extra == "ai"
+Requires-Dist: ai-parrot[agents,graphindex,images,llms,retrieval]>=1.0.5; extra == "ai"
+Requires-Dist: ai-parrot-loaders[audio,document,ebook,ml,pdf,scraping,video,web,youtube]>=1.0.5; extra == "ai"
+Requires-Dist: ai-parrot-tools[analysis,arxiv,aws,codeinterpreter,db,docker,excel,finance,flowtask,git,google,jira,messaging,pdf,pulumi,sandbox,scraping,sitesearch,slack,weather,wikipedia]>=1.0.5; extra == "ai"
+Requires-Dist: ai-parrot-pipelines>=1.0.5; extra == "ai"
+Requires-Dist: ai-parrot-server[all]>=1.0.5; extra == "ai"
+Requires-Dist: ai-parrot-integrations[all]>=1.0.5; extra == "ai"
 Provides-Extra: dev
 Requires-Dist: pytest>=5.4.0; extra == "dev"
 Requires-Dist: coverage; extra == "dev"

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/CopyToPg.py

@@ -10,7 +10,6 @@ from typing import Sequence, Union
 import pandas as pd
 from pandas.api.types import (
     is_datetime64_any_dtype,
-    is_datetime64tz_dtype,
     is_integer_dtype,
     is_float_dtype,
     is_bool_dtype
@@ -461,7 +460,7 @@ class CopyToPg(CopyTo):
                                 utc=True  # parse as UTC if a tz string is present
                             )
                         # After the coercion, act only on tz-aware columns
-                        if is_datetime64tz_dtype(self.data[col]):
+                        if isinstance(self.data[col].dtype, pd.DatetimeTZDtype):
                             # tz_localize(None) drops the tz info but keeps the *wall time*
                             # e.g. 2025-03-20 19:11:09+00:00 → 2025-03-20 19:11:09
                             self.data[col] = self.data[col].dt.tz_localize(None)

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/MarketClustering.py

@@ -9987,7 +9987,7 @@ class MarketClustering(FlowComponent):
         partition = self._partition_mask(self._data)
         total = 0
         for _ in range(max(1, max_rounds)):
-            markets = self._data.loc[idx, self._cluster_id].to_numpy()
+            markets = self._data.loc[idx, self._cluster_id].to_numpy(copy=True)
             # Full headcount (including sub-cluster rows) for capacity
             # checks — they count toward the market even though they
             # cannot be moved by this pass.

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/OpenStreetMap.py

@@ -964,6 +964,8 @@ class OpenStreetMap(FlowComponent):
                 clon = float(row[self.lon_column])
             except (TypeError, ValueError, KeyError):
                 return idx, None
+            if pd.isna(clat) or pd.isna(clon):
+                return idx, None
             query = self._build_geocode_query(row, clat, clon)
             if query is None:
                 return idx, None

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/SchedulingVisits.py

@@ -564,7 +564,7 @@ class SchedulingVisits(FlowComponent):
         `_pack_visits()` relies on to pick the ISO-week vs. monthly window.
         """
         frequency = visit_frequency
-        if cadence:
+        if isinstance(cadence, str) and cadence:
             parsed = self._parse_cadence(cadence)
             if parsed:
                 frequency = parsed[1]  # only the frequency label, not the rule
@@ -591,7 +591,7 @@ class SchedulingVisits(FlowComponent):
         # Uses _parse_cadence() which regexes the full leading integer —
         # the old ``int(cadence[0])`` parsed only the first character, so
         # ``"10xWeek"`` silently became 1 visit (defect 4 fix).
-        if cadence:
+        if isinstance(cadence, str) and cadence:
             parsed = self._parse_cadence(cadence)
             if parsed:
                 visit_rule, visit_frequency = parsed

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/ServiceScraperStores/retailers/costco.py

@@ -22,6 +22,7 @@ For a quick test run, pass `args: {limit: 10}` in the task YAML to stop
 after the first 10 warehouses found (see search()) instead of crawling
 every state.
 """
+import html as html_lib
 import json
 import re
 import time
@@ -46,6 +47,13 @@ US_STATE_ABBRS = [
 
 WAREHOUSE_LINK_RE = re.compile(r'href="(/w/-/[a-z0-9-]+/[a-z0-9-]+/(\d+))"')
 
+#: The warehouse's marketing name (e.g. "Aurora Village Warehouse") - has no
+#: relation to its mailing address city (store 106 is "Aurora Village" but
+#: sits in Shoreline, WA) and isn't in the LocalBusiness JSON-LD block at all
+#: (verified 2026-09-20 against costco.com/w/-/wa/shoreline/106) - it's only
+#: in this plain <h1>, so it needs its own regex.
+WAREHOUSE_NAME_RE = re.compile(r'<h1[^>]*>(.*?)</h1>', re.S)
+
 DAY_ORDER = [
     ("mon", "Monday"), ("tue", "Tuesday"), ("wed", "Wednesday"),
     ("thu", "Thursday"), ("fri", "Friday"), ("sat", "Saturday"),
@@ -71,6 +79,7 @@ class CostcoStrategy(RetailerStrategy):
     field_map = {
         "store_number": "store_number",
         "store_name": "store_name",
+        "warehouse_name": "warehouse_name",
         "street_address": "street_address",
         "city": "city",
         "state_code": "state_code",
@@ -127,9 +136,14 @@ class CostcoStrategy(RetailerStrategy):
         address = local_business.get("address") or {}
         geo = local_business.get("geo") or {}
         zipcode = (address.get("postalCode") or "").split("-")[0].strip() or None
+        name_match = WAREHOUSE_NAME_RE.search(html)
+        warehouse_name = html_lib.unescape(name_match.group(1)).strip() if name_match else None
         return {
             "store_number": number,
+            # Fallback only - the real name is warehouse_name below. Kept for
+            # pages where the <h1> match fails for some reason.
             "store_name": f"{address.get('addressLocality', '')} {address.get('addressRegion', '')}".strip() or None,
+            "warehouse_name": warehouse_name or None,
             "street_address": address.get("streetAddress"),
             "city": address.get("addressLocality"),
             "state_code": address.get("addressRegion"),

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/TableSchema.py

@@ -287,7 +287,7 @@ class TableSchema(QSSupport, FlowComponent):
                     **arguments,
                 )
             else:
-                self.params = {"infer_datetime_format": True}
+                self.params = {}
                 arguments = {**self.params}
                 if hasattr(self, "pd_args") and isinstance(self.pd_args, dict):
                     arguments = {**self.params, **self.pd_args}

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/TransformRows/TransformRows.py

@@ -556,7 +556,7 @@ class TransformRows(FlowComponent):
         if hasattr(elem, "clean_str"):
             df.is_copy = None
             u = df.select_dtypes(include=["object", "string"])
-            u = u.applymap(lambda x: x.strip() if isinstance(x, str) else x)
+            u = u.map(lambda x: x.strip() if isinstance(x, str) else x)
             df[u.columns] = df[u.columns].fillna("")
         if hasattr(elem, "drop_empty"):
             # First filter out those rows which

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/TransformRows/functions.py

@@ -425,7 +425,7 @@ def drop_timezone(
             column = field
 
         series = df[column]
-        if pandas.api.types.is_datetime64tz_dtype(series):
+        if isinstance(series.dtype, pandas.DatetimeTZDtype):
             # This is a regular tz-aware pandas Series
             df[field] = series.dt.tz_localize(None)
             return df
@@ -1427,7 +1427,7 @@ def to_integer(df: pandas.DataFrame, field: str):
         df[field] = pandas.to_numeric(df[field], errors="coerce")
         # Replace non-finite values (NaN, inf, -inf) with pd.NA before converting to int64
         df[field] = df[field].where(np.isfinite(df[field]), pandas.NA)
-        df[field] = df[field].astype("Int64", copy=False)
+        df[field] = df[field].astype("Int64")
     except TypeError as err:
         print(f"TO Integer {field}: Unable to safely cast non-equivalent float to int.")
         numeric_col = pandas.to_numeric(df[field], errors="coerce")
@@ -1458,7 +1458,7 @@ def to_float(
     :return: Modified pandas DataFrame with the values converted to floats.
     """
     try:
-        df[field] = pandas.to_numeric(df[field], errors="coerce").astype("float64", copy=False)
+        df[field] = pandas.to_numeric(df[field], errors="coerce").astype("float64")
     except Exception as err:
         print(f"TO Float {field}: An error occurred during conversion.")
         print(err)

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/Workday/workday.py

@@ -594,6 +594,9 @@ class Workday(FlowComponent):
                 "cost_center_id_type", "organization_id_type", "worker_id_type",
                 "exclude_all_attachments", "exclude_employees", "exclude_contingent_workers",
                 "StepName", "TaskName", "debug", "step_name", "task_name",
+                # Injected by Task for every component (tasks/task.py);
+                # Workday RaaS rejects unknown query params.
+                "storage_name",
             }
             for attr_name in dir(self):
                 if attr_name.startswith("_") or attr_name in excluded_attrs:

flowtask-5.13.0/flowtask/components/tDiff.py

@@ -0,0 +1,449 @@
+import asyncio
+import warnings
+from collections.abc import Callable
+from typing import Any, Optional, Union
+import numpy as np
+import pandas as pd
+from pandas import DataFrame
+from ..exceptions import ComponentError, ConfigError, DataNotFound
+from ..interfaces.flow import FlowComponent
+from ..interfaces.t_pandas import tPandas
+
+CHANGE_TYPES: tuple[str, ...] = ("new", "modified", "deleted", "unchanged")
+METHODS: tuple[str, ...] = ("compare", "hash")
+DTYPE_POLICIES: tuple[str, ...] = ("strict", "coerce")
+
+
+def _is_null(value: Any) -> bool:
+    """Return True when a scalar cell is null (None, NaN, pd.NA, NaT)."""
+    return bool(pd.api.types.is_scalar(value) and pd.isna(value))
+
+
+def _safe_equal(left: Any, right: Any) -> bool:
+    """Null-safe, exception-safe equality between two cells (plain equality, no nested diff)."""
+    left_null, right_null = _is_null(left), _is_null(right)
+    if left_null or right_null:
+        return left_null and right_null
+    try:
+        return bool(left == right)
+    except (ValueError, TypeError):
+        try:
+            return bool(np.array_equal(left, right))
+        except Exception:  # pragma: no cover - defensive
+            return False
+
+
+def _family(dtype: Any) -> str:
+    """Classify a dtype into a strict-compatibility family."""
+    if pd.api.types.is_bool_dtype(dtype):
+        return "bool"
+    if pd.api.types.is_integer_dtype(dtype):
+        return "int"
+    if pd.api.types.is_float_dtype(dtype):
+        return "float"
+    if pd.api.types.is_datetime64_any_dtype(dtype):
+        return "datetime"
+    if dtype == object or isinstance(dtype, pd.StringDtype):
+        return "text"
+    return str(dtype)
+
+
+class tDiff(tPandas):
+    """
+    tDiff
+
+        Overview
+
+            The tDiff class detects changes between two DataFrames that share a primary key. Rows are
+            classified as new, modified, deleted or unchanged (label column ``_change``) so downstream
+            steps can filter on them. Invalid primary keys fail explicitly, comparisons are null-safe,
+            schema/dtype mismatches are reported (never silently fixed) and "no changes" is a valid
+            result (empty typed DataFrame) unless raise_on_empty is set. The first source in depends is
+            the current DataFrame (A), the second is the previous one (B).
+
+    :widths: auto
+
+        | depends          |   Yes    | Exactly two sources: first is current (A), second is previous (B).                                 |
+        | pk               |   Yes    | Primary key column (str) or list of columns (composite key).                                       |
+        | compare_columns  |   No     | Columns whose change marks a row as modified (default: common columns minus pk and ignore).        |
+        | ignore_columns   |   No     | Columns that are never compared (e.g. updated_at, etl_ts).                                         |
+        | include          |   No     | Change types emitted: new, modified, deleted, unchanged (default: new, modified, deleted).         |
+        | change_column    |   No     | Name of the label column, defaults to _change.                                                     |
+        | changed_columns  |   No     | true adds _changed_columns (list of columns) on modified rows; a string names that column.         |
+        | method           |   No     | compare (vectorized per column, default) or hash (per-row hash over compare_columns).              |
+        | float_tolerance  |   No     | Dict with rtol and atol used with numpy.isclose on float columns (not valid with method hash).     |
+        | strip_strings    |   No     | Strip strings before comparing (returned values are never altered), defaults to false.             |
+        | case_sensitive   |   No     | When false, strings are compared with casefold(), defaults to true.                                |
+        | dtype_policy     |   No     | strict fails on incompatible dtypes; coerce casts B to the dtype of A (defaults to strict).        |
+        | raise_on_empty   |   No     | When true and the result is empty, raises DataNotFound, defaults to false.                         |
+
+    Return
+
+        A DataFrame with the columns of A (in A order) plus the label column. New, modified and unchanged
+        rows carry A values; deleted rows carry B values (A-only columns are null). A rows keep their order
+        and deleted rows follow in B order. Metrics: NEW_ROWS, MODIFIED_ROWS, DELETED_ROWS, UNCHANGED_ROWS
+        and SCHEMA_DRIFT; variables: {StepName}_NUMROWS, _NEW, _MODIFIED and _DELETED.
+
+    :consumes: dataframe
+    :produces: dataframe
+
+        Example:
+
+        ```yaml
+        tDiff:
+          depends:
+          - QueryToPandas_1
+          - QueryToPandas_2
+          pk: user_id
+          compare_columns:
+          - nombre
+          - email
+          include:
+          - new
+          - modified
+          - deleted
+        ```
+    """
+    _version = "1.0.0"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs: Any,
+    ) -> None:
+        """Set defaults for every property before kwargs override them."""
+        self.pk: Union[str, list[str], None] = None
+        self.compare_columns: Optional[list[str]] = None
+        self.ignore_columns: list[str] = []
+        self.include: list[str] = ["new", "modified", "deleted"]
+        self.change_column: str = "_change"
+        self.changed_columns: Union[bool, str] = False
+        self.method: str = "compare"
+        self.float_tolerance: Optional[dict[str, float]] = None
+        self.strip_strings: bool = False
+        self.case_sensitive: bool = True
+        self.dtype_policy: str = "strict"
+        self.raise_on_empty: bool = False
+        self._pk: list[str] = []
+        self._schema_drift: dict[str, list[str]] = {
+            "only_in_current": [],
+            "only_in_previous": [],
+        }
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs: Any) -> bool:
+        """Resolve current (A) / previous (B) and validate config.
+
+        Raises:
+            ComponentError: missing/extra source.
+            ConfigError: invalid include/method/dtype_policy, missing pk, hash+tolerance.
+        """
+        depends = getattr(self, "depends", None)
+        if not depends or not isinstance(depends, (list, tuple)):
+            raise ComponentError("tDiff requires 'depends' with exactly 2 sources (current, previous)")
+        if len(depends) != 2:
+            missing = "previous" if len(depends) < 2 else "extra"
+            raise ComponentError(
+                f"tDiff requires exactly 2 sources in depends (current, previous); "
+                f"got {len(depends)} ({missing} source problem)"
+            )
+        sources = self.previous
+        if not self._multi or not isinstance(sources, list) or len(sources) != 2:
+            raise ComponentError(
+                f"tDiff could not resolve both sources {list(depends)!s}: the 'previous' source is missing"
+            )
+        frames = []
+        for label, name, source in zip(("current", "previous"), depends, sources):
+            df = source.output()
+            if not isinstance(df, DataFrame):
+                raise ComponentError(f"tDiff {label} source '{name}' did not produce a DataFrame")
+            frames.append(df)
+        self.df1, self.df2 = frames
+
+        # config values
+        if self.method not in METHODS:
+            raise ConfigError(f"tDiff: invalid method '{self.method}', valid values: {list(METHODS)}")
+        if self.dtype_policy not in DTYPE_POLICIES:
+            raise ConfigError(
+                f"tDiff: invalid dtype_policy '{self.dtype_policy}', valid values: {list(DTYPE_POLICIES)}"
+            )
+        if isinstance(self.include, str):
+            self.include = [self.include]
+        invalid = [x for x in self.include if x not in CHANGE_TYPES]
+        if invalid:
+            raise ConfigError(f"tDiff: invalid include value(s) {invalid}, valid values: {list(CHANGE_TYPES)}")
+        if self.method == "hash" and self.float_tolerance:
+            raise ConfigError("tDiff: method 'hash' is incompatible with float_tolerance")
+        if not self.pk:
+            raise ConfigError("tDiff: 'pk' is required (column name or list of columns)")
+        self._pk = [self.pk] if isinstance(self.pk, str) else list(self.pk)
+        if isinstance(self.compare_columns, str):
+            self.compare_columns = [self.compare_columns]
+        if isinstance(self.ignore_columns, str):
+            self.ignore_columns = [self.ignore_columns]
+        await FlowComponent.start(self, **kwargs)
+        return True
+
+    async def close(self) -> None:
+        """No resources to release."""
+
+    def _validate_pk(self, df: DataFrame, side: str) -> None:
+        """Raise ConfigError if a pk column is missing; ComponentError on null or duplicated keys."""
+        for col in self._pk:
+            if col not in df.columns:
+                raise ConfigError(f"tDiff: pk column '{col}' is missing in {side} DataFrame")
+        nulls = int(df[self._pk].isna().any(axis=1).sum())
+        if nulls:
+            raise ComponentError(f"tDiff: pk has nulls in {side} DataFrame ({nulls} rows)")
+        dup = df.duplicated(subset=self._pk, keep=False)
+        if dup.any():
+            keys = df.loc[dup, self._pk].drop_duplicates().head(5)
+            if len(self._pk) == 1:
+                examples = keys[self._pk[0]].tolist()
+            else:
+                examples = [tuple(row) for row in keys.itertuples(index=False, name=None)]
+            raise ComponentError(
+                f"tDiff: pk is duplicated in {side} DataFrame ({int(dup.sum())} rows), examples: {examples}"
+            )
+
+    def _resolve_compare_columns(self, a: DataFrame, b: DataFrame) -> list[str]:
+        """Return the compared columns and store SCHEMA_DRIFT in self._schema_drift."""
+        ignore = set(self.ignore_columns or [])
+        only_a = [c for c in a.columns if c not in b.columns]
+        only_b = [c for c in b.columns if c not in a.columns]
+        self._schema_drift = {"only_in_current": only_a, "only_in_previous": only_b}
+        if only_a or only_b:
+            self._logger.warning(
+                f"tDiff schema drift: only in current={only_a}, only in previous={only_b} (not compared)"
+            )
+        if self.compare_columns:
+            for col in self.compare_columns:
+                if col not in a.columns:
+                    raise ConfigError(f"tDiff: compare column '{col}' is missing in current DataFrame")
+                if col not in b.columns:
+                    raise ConfigError(f"tDiff: compare column '{col}' is missing in previous DataFrame")
+            return [c for c in self.compare_columns if c not in self._pk and c not in ignore]
+        return [c for c in a.columns if c in b.columns and c not in self._pk and c not in ignore]
+
+    def _align_dtypes(self, ca: DataFrame, cb: DataFrame) -> DataFrame:
+        """Return a copy of cb aligned to ca per dtype_policy."""
+        out = cb.copy()
+        for col in ca.columns:
+            da, db = ca[col].dtype, cb[col].dtype
+            fa, fb = _family(da), _family(db)
+            if fa == "datetime" and fb == "datetime":
+                tz_a, tz_b = getattr(da, "tz", None), getattr(db, "tz", None)
+                if (tz_a is None) != (tz_b is None):
+                    raise ComponentError(f"tDiff: column '{col}' mixes tz-aware and naive datetimes ({da} vs {db})")
+                if str(tz_a) != str(tz_b):
+                    raise ComponentError(f"tDiff: column '{col}' has different timezones ({da} vs {db})")
+            if fa != fb:
+                if self.dtype_policy == "strict":
+                    raise ComponentError(f"tDiff: incompatible dtypes in column '{col}': {da} vs {db}")
+                if fa == "datetime" and fb in ("datetime", "text") and getattr(da, "tz", None) is not None:
+                    raise ComponentError(f"tDiff: column '{col}' cannot be coerced to tz-aware {da}")
+                try:
+                    out[col] = self._coerce(cb[col], da, fa)
+                except (ValueError, TypeError, OverflowError) as err:
+                    raise ComponentError(f"tDiff: cannot coerce column '{col}' ({db} -> {da}): {err!s}") from err
+            elif da != db:
+                try:
+                    out[col] = cb[col].astype(da)
+                except (ValueError, TypeError):
+                    # compatible family but not representable (e.g. nulls into int64): keep as is
+                    pass
+        return out
+
+    @staticmethod
+    def _coerce(series: pd.Series, dtype: Any, family: str) -> pd.Series:
+        """Cast a series to the dtype of A (raises ValueError/TypeError on failure)."""
+        if family in ("int", "float"):
+            return pd.to_numeric(series, errors="raise").astype(dtype)
+        if family == "datetime":
+            return pd.to_datetime(series, errors="raise").astype(dtype)
+        return series.astype(dtype)
+
+    def _normalize(self, df: DataFrame) -> DataFrame:
+        """Return a copy with strip/casefold applied to string-like columns."""
+        out = df.copy()
+        if not self.strip_strings and self.case_sensitive:
+            return out
+
+        def _fix(value: Any) -> Any:
+            if isinstance(value, str):
+                if self.strip_strings:
+                    value = value.strip()
+                if not self.case_sensitive:
+                    value = value.casefold()
+            return value
+
+        for col in out.columns:
+            if _family(out[col].dtype) == "text":
+                out[col] = out[col].astype(object).map(_fix)
+        return out
+
+    async def run(self) -> DataFrame:
+        """Run the diff; empty is a valid result unless raise_on_empty."""
+        try:
+            df = await self._run()
+        except (ComponentError, ConfigError, DataNotFound):
+            raise
+        except (ValueError, KeyError) as err:
+            raise ComponentError(f"tDiff Error: {err!s}") from err
+        if df.empty and self.raise_on_empty:
+            raise DataNotFound("tDiff: no changes found for the selected include types")
+        self._result = df
+        self.add_metric("NUM_ROWS", df.shape[0])
+        self.add_metric("NUM_COLUMNS", df.shape[1])
+        return self._result
+
+    async def _run(self) -> DataFrame:
+        """Classify rows into new/modified/deleted/unchanged and assemble the output."""
+        a, b = self.df1, self.df2
+        pk = self._pk
+        self._validate_pk(a, "current")
+        self._validate_pk(b, "previous")
+        cols = self._resolve_compare_columns(a, b)
+
+        # keys (pk dtypes must be compatible too)
+        b_pk = self._align_dtypes(a[pk], b[pk])
+        if len(pk) == 1:
+            a_keys = pd.Index(a[pk[0]])
+            b_keys = pd.Index(b_pk[pk[0]])
+        else:
+            a_keys = pd.MultiIndex.from_frame(a[pk])
+            b_keys = pd.MultiIndex.from_frame(b_pk)
+        new_mask = ~np.asarray(a_keys.isin(b_keys))
+        deleted_pos = np.flatnonzero(~np.asarray(b_keys.isin(a_keys)))
+        b_pos = np.asarray(b_keys.get_indexer(a_keys))
+        common_a = np.flatnonzero(b_pos >= 0)
+        common_b = b_pos[common_a]
+
+        # compared copies
+        ca = a[cols].iloc[common_a].reset_index(drop=True)
+        cb_raw = b[cols].iloc[common_b].reset_index(drop=True)
+        cb = self._align_dtypes(ca, cb_raw)
+        ca, cb = self._normalize(ca), self._normalize(cb)
+
+        eq: Optional[DataFrame] = None
+        if not cols or len(common_a) == 0:
+            modified = np.zeros(len(common_a), dtype=bool)
+        elif self.method == "hash":
+            modified = np.asarray(self._hash_modified(ca, cb), dtype=bool)
+        else:
+            eq = self._equal_matrix(ca, cb)
+            modified = ~eq.all(axis=1).to_numpy(dtype=bool)
+
+        labels = np.full(len(a), "new", dtype=object)
+        labels[common_a] = np.where(modified, "modified", "unchanged")
+
+        # changed columns (only for modified rows)
+        cc_name: Optional[str] = None
+        if self.changed_columns:
+            cc_name = self.changed_columns if isinstance(self.changed_columns, str) else "_changed_columns"
+        changed_lists: Optional[list[list[str]]] = None
+        if cc_name is not None:
+            changed_lists = [[] for _ in range(len(a))]
+            if modified.any():
+                if eq is None:
+                    eq_mod = self._equal_matrix(
+                        ca[modified].reset_index(drop=True), cb[modified].reset_index(drop=True)
+                    )
+                else:
+                    eq_mod = eq[modified]
+                flags = ~eq_mod.to_numpy(dtype=bool)
+                for pos, row in zip(common_a[modified], flags):
+                    changed_lists[pos] = [c for c, diff in zip(cols, row) if diff]
+
+        # metrics (independent of include)
+        n_mod = int(modified.sum())
+        n_new = int(new_mask.sum())
+        n_del = int(len(deleted_pos))
+        n_unch = int(len(common_a) - n_mod)
+        self.add_metric("NEW_ROWS", n_new)
+        self.add_metric("MODIFIED_ROWS", n_mod)
+        self.add_metric("DELETED_ROWS", n_del)
+        self.add_metric("UNCHANGED_ROWS", n_unch)
+        self.add_metric("SCHEMA_DRIFT", self._schema_drift)
+
+        # assemble
+        keep = np.flatnonzero(np.isin(labels, [x for x in self.include if x != "deleted"]))
+        parts: list[DataFrame] = []
+        if len(keep):
+            part = a.iloc[keep].copy()
+            part[self.change_column] = labels[keep]
+            if cc_name is not None:
+                part[cc_name] = pd.Series([changed_lists[i] for i in keep], index=part.index, dtype=object)
+            parts.append(part)
+        if "deleted" in self.include and n_del:
+            dele = b.iloc[deleted_pos].reindex(columns=a.columns).copy()
+            dele[self.change_column] = "deleted"
+            if cc_name is not None:
+                dele[cc_name] = pd.Series([[] for _ in range(n_del)], index=dele.index, dtype=object)
+            parts.append(dele)
+        if not parts:
+            out = self._empty_output(a)
+        elif len(parts) == 1:
+            out = parts[0].reset_index(drop=True)
+        else:
+            with warnings.catch_warnings():
+                # A-only columns of deleted rows are all-NA: pandas upcasting is accepted (spec §8)
+                warnings.simplefilter("ignore", FutureWarning)
+                out = pd.concat(parts, ignore_index=True)
+        self._variables[f"{self.StepName}_NUMROWS"] = len(out)
+        self._variables[f"{self.StepName}_NEW"] = n_new
+        self._variables[f"{self.StepName}_MODIFIED"] = n_mod
+        self._variables[f"{self.StepName}_DELETED"] = n_del
+        return out
+
+    def _equal_matrix(self, ca: DataFrame, cb: DataFrame) -> DataFrame:
+        """Boolean frame, True where values are equal under spec §2 semantics."""
+        tol = self.float_tolerance or None
+        result: dict[str, np.ndarray] = {}
+        for col in ca.columns:
+            sa, sb = ca[col], cb[col]
+            nulls = sa.isna().to_numpy() & sb.isna().to_numpy()
+            if tol and pd.api.types.is_float_dtype(sa.dtype) and pd.api.types.is_float_dtype(sb.dtype):
+                va = sa.to_numpy(dtype=float, na_value=np.nan)
+                vb = sb.to_numpy(dtype=float, na_value=np.nan)
+                same = np.isclose(va, vb, rtol=tol.get("rtol", 1e-5), atol=tol.get("atol", 1e-8))
+            elif sa.dtype == object or sb.dtype == object:
+                try:
+                    same = (sa == sb).to_numpy(dtype=bool, na_value=False)
+                except (ValueError, TypeError):
+                    same = np.fromiter((_safe_equal(x, y) for x, y in zip(sa, sb)), dtype=bool, count=len(sa))
+            else:
+                same = (sa == sb).to_numpy(dtype=bool, na_value=False)
+            result[col] = same | nulls
+        return pd.DataFrame(result, index=ca.index, columns=list(ca.columns))
+
+    def _hash_modified(self, ca: DataFrame, cb: DataFrame) -> pd.Series:
+        """Per-row hash comparison; True where the row changed."""
+        try:
+            ha = pd.util.hash_pandas_object(ca, index=False)
+            hb = pd.util.hash_pandas_object(cb, index=False)
+        except TypeError as err:
+            bad = None
+            for col in ca.columns:
+                try:
+                    pd.util.hash_pandas_object(ca[col], index=False)
+                    pd.util.hash_pandas_object(cb[col], index=False)
+                except TypeError:
+                    bad = col
+                    break
+            where = f" in column '{bad}'" if bad else ""
+            raise ComponentError(
+                f"tDiff hash method cannot hash values{where}: {err!s}. Use method: compare for nested cells"
+            ) from err
+        return pd.Series(ha.to_numpy() != hb.to_numpy(), index=ca.index)
+
+    def _empty_output(self, a: DataFrame) -> DataFrame:
+        """Zero-row frame with A's columns/dtypes plus the label (and changed-columns) column."""
+        out = a.iloc[0:0].copy()
+        out[self.change_column] = pd.Series([], dtype=object)
+        if self.changed_columns:
+            name = self.changed_columns if isinstance(self.changed_columns, str) else "_changed_columns"
+            out[name] = pd.Series([], dtype=object)
+        return out.reset_index(drop=True)

{flowtask-5.12.18 → flowtask-5.13.0}/flowtask/components/tJoin.py

@@ -24,6 +24,7 @@ class tJoin(FlowComponent):
         | type             |   No     | "left"    | The type of join to perform. Supported values are "left", "right", "inner",           |
         |                  |          |           | "outer", and "anti-join". When "anti-join" is used, it returns the difference         |
         |                  |          |           | of B - A, i.e., all rows present in df1 but not in df2.                               |
+        |                  |          |           | For change detection (new/modified/deleted rows) use tDiff instead.                   |
         | depends          |   Yes    | A list of dependencies defining the sources for the join.                                         |
         | operator         |   No     | The logical operator to use for join conditions, defaults to "and".                               |
         | fk               |   No     | The foreign key or list of keys to use for joining DataFrames.                                    |