splink 4.0.12.dev1__tar.gz → 4.0.14__tar.gz

{splink-4.0.12.dev1 → splink-4.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splink
-Version: 4.0.12.dev1
+Version: 4.0.14
 Summary: Fast probabilistic data linkage at scale
 Project-URL: Homepage, https://github.com/moj-analytical-services/splink
 Project-URL: Repository, https://github.com/moj-analytical-services/splink
@@ -201,6 +201,8 @@ To find the best place to ask a question, report a bug or get general advice, pl
 
 🥇 Civil Service Awards 2025: Innovation category - [Winner](https://x.com/CSWnews/status/1998488787433979981)
 
+🥇 Civil Service Awards 2025: The Excellence In Delivery Award was [won](https://www.civilserviceawards.com/winners-2025/) by a dashboard powered by Splink.
+
 🥇 OpenUK Awards 2025: Open data category - [Winner](https://openuk.uk/awards/)
 
 🥈 Civil Service Awards 2023: Best Use of Data, Science, and Technology - [Runner up](https://www.civilserviceawards.com/best-use-of-data-science-and-technology-award-2/)

{splink-4.0.12.dev1 → splink-4.0.14}/README.md

@@ -172,6 +172,8 @@ To find the best place to ask a question, report a bug or get general advice, pl
 
 🥇 Civil Service Awards 2025: Innovation category - [Winner](https://x.com/CSWnews/status/1998488787433979981)
 
+🥇 Civil Service Awards 2025: The Excellence In Delivery Award was [won](https://www.civilserviceawards.com/winners-2025/) by a dashboard powered by Splink.
+
 🥇 OpenUK Awards 2025: Open data category - [Winner](https://openuk.uk/awards/)
 
 🥈 Civil Service Awards 2023: Best Use of Data, Science, and Technology - [Runner up](https://www.civilserviceawards.com/best-use-of-data-science-and-technology-award-2/)

{splink-4.0.12.dev1 → splink-4.0.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splink"
-version = "4.0.12.dev1"
+version = "4.0.14"
 description = "Fast probabilistic data linkage at scale"
 authors = [
     { name = "Robin Linacre", email = "robinlinacre@hotmail.com" },
@@ -91,6 +91,12 @@ conflicts = [
     { extra = "athena" }
   ]
 ]
+# problematic constraints, and we do not test or otherwise use this
+# we are removing in Splink 5 anyway
+exclude-dependencies = ["awswrangler"]
+
+[tool.uv.dependency-groups]
+docs = { requires-python = ">=3.13" }
 
 [build-system]
 requires = ["hatchling"]

{splink-4.0.12.dev1 → splink-4.0.14}/splink/__init__.py

@@ -56,7 +56,7 @@ def __getattr__(name):
     raise AttributeError(f"module 'splink' has no attribute '{name}'") from None
 
 
-__version__ = "4.0.12.dev1"
+__version__ = "4.0.14"
 
 
 __all__ = [

{splink-4.0.12.dev1 → splink-4.0.14}/splink/internals/blocking.py

@@ -16,7 +16,10 @@ from splink.internals.misc import ensure_is_list
 from splink.internals.pipeline import CTEPipeline
 from splink.internals.splink_dataframe import SplinkDataFrame
 from splink.internals.unique_id_concat import _composite_unique_id_from_nodes_sql
-from splink.internals.vertically_concatenate import vertically_concatenate_sql
+from splink.internals.vertically_concatenate import (
+    split_df_concat_with_tf_into_two_tables_sqls,
+    vertically_concatenate_sql,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -386,6 +389,8 @@ class ExplodingBlockingRule(BlockingRule):
         unique_id_input_column: InputColumn,
         br: BlockingRule,
         link_type: "LinkTypeLiteralType",
+        input_tablename_l: str,
+        input_tablename_r: str,
     ) -> str:
         """generates a table of the marginal id pairs from the exploded blocking rule
         i.e. pairs are only created that match this blocking rule and NOT any of
@@ -402,11 +407,6 @@ class ExplodingBlockingRule(BlockingRule):
         id_expr_l = _composite_unique_id_from_nodes_sql(unique_id_input_columns, "l")
         id_expr_r = _composite_unique_id_from_nodes_sql(unique_id_input_columns, "r")
 
-        if link_type == "two_dataset_link_only":
-            where_condition = (
-                where_condition + " and l.source_dataset < r.source_dataset"
-            )
-
         exclude_sql = self.exclude_pairs_generated_by_all_preceding_rules_sql(
             source_dataset_input_column, unique_id_input_column
         )
@@ -414,8 +414,8 @@ class ExplodingBlockingRule(BlockingRule):
             select distinct
                 {id_expr_l} as {unique_id_col.name_l},
                 {id_expr_r} as {unique_id_col.name_r}
-            from __splink__df_concat_unnested as l
-            inner join __splink__df_concat_unnested as r
+            from {input_tablename_l} as l
+            inner join {input_tablename_r} as r
             on ({br.blocking_rule_sql})
             {where_condition}
             {exclude_sql}
@@ -511,36 +511,69 @@ def materialise_exploded_id_tables(
         return []
     exploded_tables = []
 
-    pipeline = CTEPipeline()
+    first_input_df = next(iter(splink_df_dict.values()))
 
-    sql = vertically_concatenate_sql(
-        splink_df_dict,
-        salting_required=False,
-        source_dataset_input_column=source_dataset_input_column,
-    )
-    pipeline.enqueue_sql(sql, "__splink__df_concat")
-    nodes_concat = db_api.sql_pipeline_to_splink_dataframe(pipeline)
-
-    input_columns_set = set(nodes_concat.columns)
+    input_columns_set = set(first_input_df.columns)
+    if source_dataset_input_column:
+        input_columns_set.add(source_dataset_input_column)
 
     for br in exploding_blocking_rules:
-        pipeline = CTEPipeline([nodes_concat])
+        pipeline = CTEPipeline()
+
+        sql = vertically_concatenate_sql(
+            splink_df_dict,
+            salting_required=False,
+            source_dataset_input_column=source_dataset_input_column,
+        )
+        pipeline.enqueue_sql(sql, "__splink__df_concat")
         arrays_to_explode_cols = [
             br._input_column(colname) for colname in br.array_columns_to_explode
         ]
 
         other_cols = input_columns_set - set(arrays_to_explode_cols)
 
-        expl_sql = db_api.sql_dialect.explode_arrays_sql(
-            "__splink__df_concat",
-            br.array_columns_to_explode,
-            [col.name for col in other_cols],
-        )
+        if link_type == "two_dataset_link_only":
+            if source_dataset_input_column is None:
+                raise ValueError(
+                    "source_dataset_input_column is required for two_dataset_link_only"
+                )
+
+            sqls = split_df_concat_with_tf_into_two_tables_sqls(
+                "__splink__df_concat",
+                source_dataset_input_column.name,
+            )
+            pipeline.enqueue_list_of_sqls(sqls)
 
-        pipeline.enqueue_sql(
-            expl_sql,
-            "__splink__df_concat_unnested",
-        )
+            input_tablename_l = "__splink__df_concat_left_unnested"
+            input_tablename_r = "__splink__df_concat_right_unnested"
+
+            expl_sql_l = db_api.sql_dialect.explode_arrays_sql(
+                "__splink__df_concat_left",
+                br.array_columns_to_explode,
+                [col.name for col in other_cols],
+            )
+            pipeline.enqueue_sql(expl_sql_l, input_tablename_l)
+
+            expl_sql_r = db_api.sql_dialect.explode_arrays_sql(
+                "__splink__df_concat_right",
+                br.array_columns_to_explode,
+                [col.name for col in other_cols],
+            )
+            pipeline.enqueue_sql(expl_sql_r, input_tablename_r)
+        else:
+            input_tablename_l = "__splink__df_concat_unnested"
+            input_tablename_r = "__splink__df_concat_unnested"
+
+            expl_sql = db_api.sql_dialect.explode_arrays_sql(
+                "__splink__df_concat",
+                br.array_columns_to_explode,
+                [col.name for col in other_cols],
+            )
+
+            pipeline.enqueue_sql(
+                expl_sql,
+                input_tablename_l,
+            )
 
         base_name = "__splink__marginal_exploded_ids_blocking_rule"
         table_name = f"{base_name}_mk_{br.match_key}"
@@ -550,6 +583,8 @@ def materialise_exploded_id_tables(
             unique_id_input_column=unique_id_input_column,
             br=br,
             link_type=link_type,
+            input_tablename_l=input_tablename_l,
+            input_tablename_r=input_tablename_r,
         )
 
         pipeline.enqueue_sql(sql, table_name)

{splink-4.0.12.dev1 → splink-4.0.14}/splink/internals/connected_components.py

@@ -71,7 +71,7 @@ def _cc_generate_representatives_loop_cond(
 
         from {prev_representatives}
 
-    )
+    ) as rep_union
     group by old_rep
     """
 

{splink-4.0.12.dev1 → splink-4.0.14}/splink/internals/input_column.py

@@ -83,11 +83,11 @@ class SqlglotColumnTreeBuilder:
         def tree_to_sqlglot_column_tree_builder_args(sqlglot_tree, sqlglot_dialect):
             args = {"sqlglot_dialect": sqlglot_dialect, "quoted": True}
             if sqlglot_tree.find(exp.Bracket):
-                lit = sqlglot_tree.find(exp.Bracket).find(exp.Literal)
-                if lit.args["is_string"]:
-                    args["bracket_key"] = lit.args["this"]
+                lit = sqlglot_tree.find(exp.Bracket).expressions[0]
+                if lit.is_string:
+                    args["bracket_key"] = lit.this
                 else:
-                    args["bracket_index"] = int(lit.args["this"])
+                    args["bracket_index"] = int(lit.sql())
 
             args["column_name"] = sqlglot_tree.find(exp.Identifier).args["this"]
             return args
@@ -103,6 +103,9 @@ class SqlglotColumnTreeBuilder:
 
         valid_signatures = {
             sqlglot_tree_signature(sqlglot.parse_one("col_name")),
+            # negative indices are valid in certain contexts (postgres custom indexing,
+            # duckdb), and are treated separately in newer sqlglot versions (28.7.0+)
+            sqlglot_tree_signature(sqlglot.parse_one("col_name[-1]")),
             sqlglot_tree_signature(sqlglot.parse_one("col_name[1]")),
             sqlglot_tree_signature(sqlglot.parse_one("col_name['lat']")),
         }

{splink-4.0.12.dev1 → splink-4.0.14}/splink/internals/spark/database_api.py

@@ -299,6 +299,7 @@ class SparkAPI(DatabaseAPI[spark_df]):
             r"__splink__clusters_at_all_thresholds",
             r"__splink__clustering_output_final",
             r"__splink__stable_nodes_at_new_threshold",
+            r"__splink__filtered_neighbours.*",
         ]
 
         if re.fullmatch(r"|".join(regex_to_persist), templated_name):