sqlmesh 0.190.1.dev8__py3-none-any.whl → 0.191.1.dev1__py3-none-any.whl

sqlmesh/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.190.1.dev8'
-__version_tuple__ = version_tuple = (0, 190, 1, 'dev8')
+__version__ = version = '0.191.1.dev1'
+__version_tuple__ = version_tuple = (0, 191, 1, 'dev1')

sqlmesh/core/table_diff.py

@@ -24,6 +24,7 @@ if t.TYPE_CHECKING:
     from sqlmesh.core.engine_adapter import EngineAdapter
 
 SQLMESH_JOIN_KEY_COL = "__sqlmesh_join_key"
+SQLMESH_SAMPLE_TYPE_COL = "__sqlmesh_sample_type"
 
 
 class SchemaDiff(PydanticModel, frozen=True):
@@ -389,9 +390,6 @@ class TableDiff:
                 for c, t in matched_columns.items()
             ]
 
-            def name(e: exp.Expression) -> str:
-                return e.args["alias"].sql(identify=True)
-
             source_query = (
                 exp.select(
                     *(exp.column(c) for c in source_schema),
@@ -581,30 +579,19 @@ class TableDiff:
                     .drop(index=index_cols, errors="ignore")
                 )
 
-                sample_filter_cols = ["s_exists", "t_exists", "row_joined", "row_full_match"]
-                sample_query = (
-                    exp.select(
-                        *(sample_filter_cols),
-                        *(name(c) for c in s_selects.values()),
-                        *(name(c) for c in t_selects.values()),
-                    )
-                    .from_(table)
-                    .where(exp.or_(*(exp.column(c.alias).eq(0) for c in comparisons)))
-                    .order_by(
-                        *(name(s_selects[c.name]) for c in s_index),
-                        *(name(t_selects[c.name]) for c in t_index),
-                    )
-                    .limit(self.limit)
+                sample = self._fetch_sample(
+                    table, s_selects, s_index, t_selects, t_index, self.limit
                 )
-                sample = self.adapter.fetchdf(sample_query, quote_identifiers=True)
 
                 joined_sample_cols = [f"s__{c}" for c in s_index_names]
                 comparison_cols = [
                     (f"s__{c}", f"t__{c}")
                     for c in column_stats[column_stats["pct_match"] < 100].index
                 ]
+
                 for cols in comparison_cols:
                     joined_sample_cols.extend(cols)
+
                 joined_renamed_cols = {
                     c: c.split("__")[1] if c.split("__")[1] in index_cols else c
                     for c in joined_sample_cols
@@ -638,13 +625,16 @@ class TableDiff:
                         )
                         for c, n in joined_renamed_cols.items()
                     }
-                joined_sample = sample[sample["row_joined"] == 1][joined_sample_cols]
+
+                joined_sample = sample[sample[SQLMESH_SAMPLE_TYPE_COL] == "common_rows"][
+                    joined_sample_cols
+                ]
                 joined_sample.rename(
                     columns=joined_renamed_cols,
                     inplace=True,
                 )
 
-                s_sample = sample[(sample["s_exists"] == 1) & (sample["row_joined"] == 0)][
+                s_sample = sample[sample[SQLMESH_SAMPLE_TYPE_COL] == "source_only"][
                     [
                         *[f"s__{c}" for c in s_index_names],
                         *[f"s__{c}" for c in source_schema if c not in s_index_names],
@@ -654,7 +644,7 @@ class TableDiff:
                     columns={c: c.replace("s__", "") for c in s_sample.columns}, inplace=True
                 )
 
-                t_sample = sample[(sample["t_exists"] == 1) & (sample["row_joined"] == 0)][
+                t_sample = sample[sample[SQLMESH_SAMPLE_TYPE_COL] == "target_only"][
                     [
                         *[f"t__{c}" for c in t_index_names],
                         *[f"t__{c}" for c in target_schema if c not in t_index_names],
@@ -665,8 +655,11 @@ class TableDiff:
                 )
 
                 sample.drop(
-                    columns=sample_filter_cols
-                    + [f"s__{SQLMESH_JOIN_KEY_COL}", f"t__{SQLMESH_JOIN_KEY_COL}"],
+                    columns=[
+                        f"s__{SQLMESH_JOIN_KEY_COL}",
+                        f"t__{SQLMESH_JOIN_KEY_COL}",
+                        SQLMESH_SAMPLE_TYPE_COL,
+                    ],
                     inplace=True,
                 )
 
@@ -684,4 +677,75 @@ class TableDiff:
                     model_name=self.model_name,
                     decimals=self.decimals,
                 )
+
         return self._row_diff
+
+    def _fetch_sample(
+        self,
+        sample_table: exp.Table,
+        s_selects: t.Dict[str, exp.Alias],
+        s_index: t.List[exp.Column],
+        t_selects: t.Dict[str, exp.Alias],
+        t_index: t.List[exp.Column],
+        limit: int,
+    ) -> pd.DataFrame:
+        rendered_data_column_names = [
+            name(s) for s in list(s_selects.values()) + list(t_selects.values())
+        ]
+        sample_type = exp.to_identifier(SQLMESH_SAMPLE_TYPE_COL)
+
+        source_only_sample = (
+            exp.select(
+                exp.Literal.string("source_only").as_(sample_type), *rendered_data_column_names
+            )
+            .from_(sample_table)
+            .where(exp.and_(exp.column("s_exists").eq(1), exp.column("row_joined").eq(0)))
+            .order_by(*(name(s_selects[c.name]) for c in s_index))
+            .limit(limit)
+        )
+
+        target_only_sample = (
+            exp.select(
+                exp.Literal.string("target_only").as_(sample_type), *rendered_data_column_names
+            )
+            .from_(sample_table)
+            .where(exp.and_(exp.column("t_exists").eq(1), exp.column("row_joined").eq(0)))
+            .order_by(*(name(t_selects[c.name]) for c in t_index))
+            .limit(limit)
+        )
+
+        common_rows_sample = (
+            exp.select(
+                exp.Literal.string("common_rows").as_(sample_type), *rendered_data_column_names
+            )
+            .from_(sample_table)
+            .where(exp.and_(exp.column("row_joined").eq(1), exp.column("row_full_match").eq(0)))
+            .order_by(
+                *(name(s_selects[c.name]) for c in s_index),
+                *(name(t_selects[c.name]) for c in t_index),
+            )
+            .limit(limit)
+        )
+
+        query = (
+            exp.Select()
+            .with_("source_only", source_only_sample)
+            .with_("target_only", target_only_sample)
+            .with_("common_rows", common_rows_sample)
+            .select(sample_type, *rendered_data_column_names)
+            .from_("source_only")
+            .union(
+                exp.select(sample_type, *rendered_data_column_names).from_("target_only"),
+                distinct=False,
+            )
+            .union(
+                exp.select(sample_type, *rendered_data_column_names).from_("common_rows"),
+                distinct=False,
+            )
+        )
+
+        return self.adapter.fetchdf(query, quote_identifiers=True)
+
+
+def name(e: exp.Expression) -> str:
+    return e.args["alias"].sql(identify=True)

{sqlmesh-0.190.1.dev8.dist-info → sqlmesh-0.191.1.dev1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.190.1.dev8
+Version: 0.191.1.dev1
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.190.1.dev8.dist-info → sqlmesh-0.191.1.dev1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=L9Hjq6o7TO4EcAMCq8IEAuJdYn9ib5xMWhJXtVtC1DI,528
+sqlmesh/_version.py,sha256=7hh-fhvpsFAPwOfArTlM6CZ9gYvfxD3DI1WDSOp7590,528
 sqlmesh/magics.py,sha256=gjzoxREb1XT8Yho_PQMlMqhhkLq_sEfBM8S3PTgxvWE,40337
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -29,7 +29,7 @@ sqlmesh/core/schema_diff.py,sha256=Sp2MoqAngp-yTYevkLaf9jb26SuAdGXoZ54g5MjWowU,3
 sqlmesh/core/schema_loader.py,sha256=lvrI9k7hr1P4X4E6t3U-KrcuvdQuw0UdNlN90P2s52w,3563
 sqlmesh/core/selector.py,sha256=_hcktUQiKXCPE-mOqXClHUVWBfEZKMPVVNDtpSSRhbI,12660
 sqlmesh/core/signal.py,sha256=yVLXewYP6eH0vt-EwaIaI2WwHz6hWsVr-Kb2m4kP1FU,1353
-sqlmesh/core/table_diff.py,sha256=6J-C3Vr2Aw02t3VXJwFPjazao-yv92jHgYZ31Iok_6A,27133
+sqlmesh/core/table_diff.py,sha256=_LhSwAhCctkqWini0n-qeIK-cOPeuGnwlLSuUNM8n2k,29062
 sqlmesh/core/user.py,sha256=EJ6R4R1iK67n80vBoCCsidF56IR7xEYqiCEO-nrVMso,1660
 sqlmesh/core/analytics/__init__.py,sha256=ou3ZXAJfQOXEifj-PzaXwMDSvJzsVaqaMkUopiI00kM,3247
 sqlmesh/core/analytics/collector.py,sha256=sggJvER16Jrg5triTC27E2KwaQvfkVDv4D6R1lVPH5I,13058
@@ -275,7 +275,7 @@ sqlmesh/utils/pydantic.py,sha256=I898jAFqa3qU6lG2uPpRebVFncOUo8ZrLWM_R3K3DHU,116
 sqlmesh/utils/rich.py,sha256=xNZyPCqEc4TAlJlsEthg-5eYUWpaizz0QjBzO7a1Yv4,1509
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=np-WVcwmwsKghCgAj1OBBMsK1SXSRn4wR-H37cGiFr8,2282
-sqlmesh-0.190.1.dev8.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.191.1.dev1.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 web/client/dist/index.html,sha256=k5bAEoPDJ0BhYQ4hBcKWObITCxIgqOUC28nGXvEDrt0,5472
 web/client/dist/assets/Audits-Pgs8N6IE.js,sha256=BQuc9kiznZSqTnzNrJSDNqAAdOTGjNQJt32M1sHZFok,1041
@@ -393,8 +393,8 @@ web/server/api/endpoints/models.py,sha256=ud-lRwriyI_OYg9tZchrBgeiXELiRTLNEodraf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=OYuQ_4dgbhG9ibTcX1a8mvYgH3ANxw9Fhbb4wg6QWqc,9222
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.190.1.dev8.dist-info/METADATA,sha256=yqtMm5HDmlJIUXP8u3bdDnVnhOt1aqwAgYWEQqGzLq8,26540
-sqlmesh-0.190.1.dev8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.190.1.dev8.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
-sqlmesh-0.190.1.dev8.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
-sqlmesh-0.190.1.dev8.dist-info/RECORD,,
+sqlmesh-0.191.1.dev1.dist-info/METADATA,sha256=Yc9GVN9IIzZxnLpB6NEFetXNdSHxdfCwK5dzFeQtJV8,26540
+sqlmesh-0.191.1.dev1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.191.1.dev1.dist-info/entry_points.txt,sha256=pIYKtg-u-uO7BK_Y0qrdmfyj0bYrcMc3Xiqr5WE3VFk,121
+sqlmesh-0.191.1.dev1.dist-info/top_level.txt,sha256=bspjo5vjELQkuN0qKuTFIbsrUt3Zeo8D2mlKeHG1LdE,12
+sqlmesh-0.191.1.dev1.dist-info/RECORD,,