splink 5.0.0.dev1__tar.gz → 5.0.0.dev3__tar.gz

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/.gitignore

@@ -163,6 +163,8 @@ cython_debug/
 
 *.parquet
 *.csv
+*.json
+
 
 # html files opt-in only
 *.html
@@ -177,5 +179,6 @@ cython_debug/
 splink_db
 splink_db_log
 spark-warehouse
+*.ddb*
 
 scripts/pyspelling/dictionary.dic

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splink
-Version: 5.0.0.dev1
+Version: 5.0.0.dev3
 Summary: Fast probabilistic data linkage at scale
 Project-URL: Homepage, https://github.com/moj-analytical-services/splink
 Project-URL: Repository, https://github.com/moj-analytical-services/splink
@@ -13,16 +13,18 @@ Requires-Dist: altair>=5.0.1
 Requires-Dist: duckdb>=0.9.2
 Requires-Dist: igraph>=0.11.2
 Requires-Dist: jinja2>=3.0.3
-Requires-Dist: numpy>=1.19.3
-Requires-Dist: pandas>=1.3.5
 Requires-Dist: sqlglot>=17.6.0
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary>=2.9.0; extra == 'postgres'
 Requires-Dist: sqlalchemy>=2.0.0; extra == 'postgres'
+Provides-Extra: pyarrow
+Requires-Dist: pyarrow>=15.0.0; extra == 'pyarrow'
 Provides-Extra: pyspark
 Requires-Dist: pyspark>=3.5.0; extra == 'pyspark'
 Provides-Extra: spark
 Requires-Dist: pyspark>=3.5.0; extra == 'spark'
+Provides-Extra: sqlite
+Requires-Dist: rapidfuzz>=3.10.0; extra == 'sqlite'
 Description-Content-Type: text/markdown
 
 <p align="center">
@@ -194,6 +196,8 @@ To find the best place to ask a question, report a bug or get general advice, pl
 
 🥇 Civil Service Awards 2025: Innovation category - [Winner](https://x.com/CSWnews/status/1998488787433979981)
 
+🥇 Civil Service Awards 2025: The Excellence In Delivery Award was [won](https://www.civilserviceawards.com/winners-2025/) by a dashboard powered by Splink.
+
 🥇 OpenUK Awards 2025: Open data category - [Winner](https://openuk.uk/awards/)
 
 🥈 Civil Service Awards 2023: Best Use of Data, Science, and Technology - [Runner up](https://www.civilserviceawards.com/best-use-of-data-science-and-technology-award-2/)

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/README.md

@@ -167,6 +167,8 @@ To find the best place to ask a question, report a bug or get general advice, pl
 
 🥇 Civil Service Awards 2025: Innovation category - [Winner](https://x.com/CSWnews/status/1998488787433979981)
 
+🥇 Civil Service Awards 2025: The Excellence In Delivery Award was [won](https://www.civilserviceawards.com/winners-2025/) by a dashboard powered by Splink.
+
 🥇 OpenUK Awards 2025: Open data category - [Winner](https://openuk.uk/awards/)
 
 🥈 Civil Service Awards 2023: Best Use of Data, Science, and Technology - [Runner up](https://www.civilserviceawards.com/best-use-of-data-science-and-technology-award-2/)

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splink"
-version = "5.0.0.dev1"
+version = "5.0.0.dev3"
 description = "Fast probabilistic data linkage at scale"
 authors = [
     { name = "Robin Linacre", email = "robinlinacre@hotmail.com" },
@@ -14,19 +14,21 @@ requires-python = ">=3.9.0,<4.0.0"
 readme = "README.md"
 license = "MIT"
 dependencies = [
-    "pandas>=1.3.5",
     "duckdb>=0.9.2",
     "sqlglot>=17.6.0",
     "altair>=5.0.1",
     "Jinja2>=3.0.3",
-    "numpy>=1.19.3",
     "igraph>=0.11.2",
 ]
 
 [project.optional-dependencies]
 pyspark = ["pyspark>=3.5.0"]
 spark = ["pyspark>=3.5.0"]
+# sqlite doesn't _require_ this functionality, but it's used if we want to use fuzzy levels
+sqlite = ["rapidfuzz>=3.10.0"]
 postgres = ["sqlalchemy>=2.0.0", "psycopg2-binary>=2.9.0"]
+pyarrow = ["pyarrow>=15.0.0"]
+# TODO: pandas optional
 
 [project.urls]
 Homepage = "https://github.com/moj-analytical-services/splink"
@@ -35,18 +37,24 @@ Repository = "https://github.com/moj-analytical-services/splink"
 [dependency-groups]
 # allows us to easily check package functionality with just these installed
 # (and not the rest of our required dependencies)
-core = ["sqlglot", "pandas"]
+core = ["sqlglot", "duckdb"]
 demos = [
     "ipywidgets>=8.1.7",
+    "jupytext>=1.18.1",
     "nbmake>=1.5.5",
+    "pseudopeople>=1.2.6",
     "pytest-xdist>=3.6.1",
 ]
+# dev deliberately doesn't include spark, so we can opt in to different spark versions
 dev = [
-    "tabulate>=0.8.9",
-    "pyspark>=3.5.0",
-    "sqlalchemy>=2.0.0",
-    "psycopg2-binary>=2.9.0",
+    { "include-group" = "linting" },
+    { "include-group" = "testing-core" },
+    { "include-group" = "typechecking" },
+]
+spark-3 = [
+  "pyspark<4; python_version < '3.12'",
 ]
+spark-4 = ["pyspark>=4.0.0"]
 docs = [
     "griffe>=1.4.0",
     # keep jinja2 relaxed so it doesn't constrain us in rest of dev
@@ -65,24 +73,49 @@ docs = [
     "pymdown-extensions>=10.15",
 ]
 linting = ["ruff>=0.4.2,<0.5"]
-testing = [
+pandas = ["pandas>=1.3.5", "numpy>=1.19.3"]
+postgres = [
+    "psycopg2-binary>=2.9.0",
+    "sqlalchemy>=2.0.0",
+]
+sqlite = [
+    "rapidfuzz>=3.10.0",
+]
+testing-core = [
     "pytest>=7.3",
-    "pyarrow>=7.0.0",
+    "pyarrow>=15.0.0",
     "networkx>=2.5.1",
-    "rapidfuzz>=3.0.0",
     "pytest-cov>=5.0.0",
+    { "include-group" = "postgres" },
+    { "include-group" = "sqlite" },
+    # TODO: temporary measure until we can depanda tests
+    { "include-group" = "pandas" },
+]
+testing = [
+    { "include-group" = "testing-core" },
+    { "include-group" = "spark-4" },
 ]
 typechecking = ["mypy==1.9.0"]
+# we will migrate to ty, but won't happen all at once, so run in parallel
+typechecking-dev = [
+    "ty>=0.0.21",
+]
 
 [tool.uv]
 default-groups = [
-    "core",
     "dev",
-    "linting",
-    "testing",
-    "typechecking",
+    "spark-4"
+]
+conflicts = [
+  [
+    { group = "spark-4" },
+    { group = "spark-3" },
+  ]
 ]
 
+[tool.uv.dependency-groups]
+docs = { requires-python = ">=3.13" }
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
@@ -115,6 +148,13 @@ lint.ignore = [
     "B006", # Do not use mutable data structures for argument defaults"
 ]
 
+# really we want to lint docs with a looser set of rules (such as not enforcing the same line length)
+# for now we simply exclude
+exclude = [
+    "docs",
+]
+
+
 [tool.pytest.ini_options]
 addopts = ["-m default"]
 markers = [
@@ -132,6 +172,8 @@ markers = [
     "sqlite_only",
     "postgres",
     "postgres_only",
+# optional dependendencies beyond backends
+    "needs_pandas",
 ]
 
 [tool.mypy]

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/splink/__init__.py

@@ -56,7 +56,7 @@ def __getattr__(name):
     raise AttributeError(f"module 'splink' has no attribute '{name}'") from None
 
 
-__version__ = "5.0.0.dev1"
+__version__ = "5.0.0.dev3"
 
 
 __all__ = [

splink-5.0.0.dev3/splink/backends/duckdb.py

@@ -0,0 +1,6 @@
+from splink.internals.duckdb.database_api import DuckDBAPI
+from splink.internals.duckdb.database_api_with_profiling import (
+    DuckDBAPIWithProfiling,
+)
+
+__all__ = ["DuckDBAPI", "DuckDBAPIWithProfiling"]

splink-5.0.0.dev3/splink/backends/spark.py

@@ -0,0 +1,5 @@
+from splink.internals.spark.database_api import SparkAPI
+from splink.internals.spark.database_api_with_profiling import SparkAPIWithProfiling
+from splink.internals.spark.jar_location import similarity_jar_location
+
+__all__ = ["similarity_jar_location", "SparkAPI", "SparkAPIWithProfiling"]

splink-5.0.0.dev3/splink/exploratory.py

@@ -0,0 +1,4 @@
+from .internals.completeness import completeness_chart
+from .internals.profile_data import profile_columns
+
+__all__ = ["completeness_chart", "profile_columns"]

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/splink/internals/accuracy.py

@@ -14,8 +14,8 @@ from splink.internals.predict import predict_from_comparison_vectors_sqls_using_
 from splink.internals.splink_dataframe import SplinkDataFrame
 from splink.internals.sql_transform import move_l_r_table_prefix_to_column_suffix
 from splink.internals.vertically_concatenate import (
-    compute_df_concat,
-    compute_df_concat_with_tf,
+    enqueue_df_concat,
+    enqueue_df_concat_with_tf,
 )
 
 if TYPE_CHECKING:
@@ -316,9 +316,7 @@ def truth_space_table_from_labels_table(
     match_weight_round_to_nearest: Optional[float] = None,
 ) -> SplinkDataFrame:
     pipeline = CTEPipeline()
-
-    nodes_with_tf = compute_df_concat_with_tf(linker, pipeline)
-    pipeline = CTEPipeline([nodes_with_tf])
+    enqueue_df_concat_with_tf(linker, pipeline)
 
     sqls = predictions_from_sample_of_pairwise_labels_sql(linker, labels_tablename)
     pipeline.enqueue_list_of_sqls(sqls)
@@ -351,13 +349,11 @@ def truth_space_table_from_labels_column(
         group_by_statement = "group by source_dataset"
 
     pipeline = CTEPipeline()
-    concat = compute_df_concat(linker, pipeline)
-
-    pipeline = CTEPipeline([concat])
+    enqueue_df_concat(linker, pipeline)
 
     sql = f"""
         select count(*) as count
-        from {concat.physical_name}
+        from __splink__df_concat
         {group_by_statement}
     """
 
@@ -451,8 +447,7 @@ def prediction_errors_from_labels_table(
     threshold_match_probability: float = 0.5,
 ) -> SplinkDataFrame:
     pipeline = CTEPipeline()
-    nodes_with_tf = compute_df_concat_with_tf(linker, pipeline)
-    pipeline = CTEPipeline([nodes_with_tf])
+    enqueue_df_concat_with_tf(linker, pipeline)
 
     sqls = predictions_from_sample_of_pairwise_labels_sql(linker, labels_tablename)
 

{splink-5.0.0.dev1 → splink-5.0.0.dev3}/splink/internals/blocking.py

@@ -14,12 +14,17 @@ from splink.internals.chunking import _chunk_assignment_sql
 from splink.internals.database_api import DatabaseAPISubClass
 from splink.internals.dialects import SplinkDialect
 from splink.internals.input_column import InputColumn
-from splink.internals.misc import ensure_is_list
+from splink.internals.misc import (
+    dedupe_preserving_order,
+    ensure_is_list,
+    join_sql_with_union_all,
+)
+from splink.internals.parse_sql import parse_columns_in_sql
 from splink.internals.pipeline import CTEPipeline
 from splink.internals.splink_dataframe import SplinkDataFrame
 from splink.internals.unique_id_concat import _composite_unique_id_from_nodes_sql
 from splink.internals.vertically_concatenate import (
-    split_df_concat_with_tf_into_two_tables_sqls,
+    select_two_dataset_link_only_input_tables_sqls,
     vertically_concatenate_sql,
 )
 
@@ -76,6 +81,24 @@ def combine_unique_id_input_columns(
     return unique_id_input_columns
 
 
+def _columns_needed_for_blocking(
+    blocking_rules: List["BlockingRule"],
+    source_dataset_input_column: Optional[InputColumn],
+    unique_id_input_column: InputColumn,
+) -> List[InputColumn]:
+    input_columns = combine_unique_id_input_columns(
+        source_dataset_input_column, unique_id_input_column
+    )
+
+    for br in blocking_rules:
+        parsed_columns = parse_columns_in_sql(
+            br.blocking_rule_sql, sqlglot_dialect=br.sqlglot_dialect
+        )
+        input_columns.extend(br._input_column(col.name) for col in parsed_columns)
+
+    return dedupe_preserving_order(input_columns)
+
+
 class BlockingRule:
     def __init__(
         self,
@@ -98,6 +121,10 @@ class BlockingRule:
     def sqlglot_dialect(self) -> str:
         return SplinkDialect.from_string(self._sql_dialect_str).sqlglot_dialect
 
+    @property
+    def requires_blocking_input_materialisation(self) -> bool:
+        return False
+
     @property
     def sql_dialect(self) -> SplinkDialect:
         return SplinkDialect.from_string(self._sql_dialect_str)
@@ -319,6 +346,8 @@ class ExplodingBlockingRule(BlockingRule):
         unique_id_input_column: InputColumn,
         br: BlockingRule,
         link_type: "LinkTypeLiteralType",
+        input_tablename_l: str,
+        input_tablename_r: str,
         left_chunk: tuple[int, int] | None = None,
         right_chunk: tuple[int, int] | None = None,
     ) -> str:
@@ -349,11 +378,6 @@ class ExplodingBlockingRule(BlockingRule):
         id_expr_l = _composite_unique_id_from_nodes_sql(unique_id_input_columns, "l")
         id_expr_r = _composite_unique_id_from_nodes_sql(unique_id_input_columns, "r")
 
-        if link_type == "two_dataset_link_only":
-            where_condition = (
-                where_condition + " and l.source_dataset < r.source_dataset"
-            )
-
         exclude_sql = self.exclude_pairs_generated_by_all_preceding_rules_sql(
             source_dataset_input_column, unique_id_input_column
         )
@@ -361,8 +385,8 @@ class ExplodingBlockingRule(BlockingRule):
             select distinct
                 {id_expr_l} as {unique_id_col.name_l},
                 {id_expr_r} as {unique_id_col.name_r}
-            from __splink__df_concat_unnested as l
-            inner join __splink__df_concat_unnested as r
+            from {input_tablename_l} as l
+            inner join {input_tablename_r} as r
             on ({br.blocking_rule_sql})
             {where_condition}
             {exclude_sql}
@@ -387,6 +411,10 @@ class ExplodingBlockingRule(BlockingRule):
 
         return "false"
 
+    @property
+    def requires_blocking_input_materialisation(self) -> bool:
+        return True
+
     def create_blocked_pairs_sql(
         self,
         *,
@@ -468,34 +496,65 @@ def materialise_exploded_id_tables(
         return []
     exploded_tables = []
 
-    pipeline = CTEPipeline()
-
-    sql = vertically_concatenate_sql(
-        splink_df_dict, source_dataset_input_column=source_dataset_input_column
-    )
-    pipeline.enqueue_sql(sql, "__splink__df_concat")
-    nodes_concat = db_api.sql_pipeline_to_splink_dataframe(pipeline)
-
-    input_columns_set = set(nodes_concat.columns)
-
     for br in exploding_blocking_rules:
-        pipeline = CTEPipeline([nodes_concat])
+        pipeline = CTEPipeline()
         arrays_to_explode_cols = [
             br._input_column(colname) for colname in br.array_columns_to_explode
         ]
+        input_columns = _columns_needed_for_blocking(
+            [*br.preceding_rules, br],
+            source_dataset_input_column=source_dataset_input_column,
+            unique_id_input_column=unique_id_input_column,
+        )
+        other_cols = [col for col in input_columns if col not in arrays_to_explode_cols]
 
-        other_cols = input_columns_set - set(arrays_to_explode_cols)
+        if link_type == "two_dataset_link_only":
+            if source_dataset_input_column is None:
+                raise ValueError(
+                    "source_dataset_input_column is required for two_dataset_link_only"
+                )
+
+            left_sql, right_sql = select_two_dataset_link_only_input_tables_sqls(
+                splink_df_dict,
+                input_columns=input_columns,
+                source_dataset_input_column=source_dataset_input_column,
+            )
 
-        expl_sql = db_api.sql_dialect.explode_arrays_sql(
-            "__splink__df_concat",
-            br.array_columns_to_explode,
-            [col.name for col in other_cols],
-        )
+            input_tablename_l = "__splink__df_concat_left_unnested"
+            input_tablename_r = "__splink__df_concat_right_unnested"
 
-        pipeline.enqueue_sql(
-            expl_sql,
-            "__splink__df_concat_unnested",
-        )
+            expl_sql_l = db_api.sql_dialect.explode_arrays_sql(
+                f"({left_sql})",
+                br.array_columns_to_explode,
+                [col.name for col in other_cols],
+            )
+            pipeline.enqueue_sql(expl_sql_l, input_tablename_l)
+
+            expl_sql_r = db_api.sql_dialect.explode_arrays_sql(
+                f"({right_sql})",
+                br.array_columns_to_explode,
+                [col.name for col in other_cols],
+            )
+            pipeline.enqueue_sql(expl_sql_r, input_tablename_r)
+        else:
+            sql = vertically_concatenate_sql(
+                splink_df_dict, source_dataset_input_column=source_dataset_input_column
+            )
+            pipeline.enqueue_sql(sql, "__splink__df_concat")
+
+            input_tablename_l = "__splink__df_concat_unnested"
+            input_tablename_r = "__splink__df_concat_unnested"
+
+            expl_sql = db_api.sql_dialect.explode_arrays_sql(
+                "__splink__df_concat",
+                br.array_columns_to_explode,
+                [col.name for col in other_cols],
+            )
+
+            pipeline.enqueue_sql(
+                expl_sql,
+                input_tablename_l,
+            )
 
         base_name = "__splink__marginal_exploded_ids_blocking_rule"
         table_name = f"{base_name}_mk_{br.match_key}"
@@ -505,6 +564,8 @@ def materialise_exploded_id_tables(
             unique_id_input_column=unique_id_input_column,
             br=br,
             link_type=link_type,
+            input_tablename_l=input_tablename_l,
+            input_tablename_r=input_tablename_r,
             left_chunk=left_chunk,
             right_chunk=right_chunk,
         )
@@ -559,11 +620,18 @@ def compute_blocked_pairs_from_concat_with_tf(
                 "source_dataset column"
             )
 
-        sqls = split_df_concat_with_tf_into_two_tables_sqls(
-            df_concat_with_tf_table_name,
-            source_dataset_input_column.name,
+        input_columns = _columns_needed_for_blocking(
+            blocking_rules=blocking_rules,
+            source_dataset_input_column=source_dataset_input_column,
+            unique_id_input_column=unique_id_input_column,
+        )
+        left_sql, right_sql = select_two_dataset_link_only_input_tables_sqls(
+            splink_df_dict,
+            input_columns=input_columns,
+            source_dataset_input_column=source_dataset_input_column,
         )
-        pipeline.enqueue_list_of_sqls(sqls)
+        pipeline.enqueue_sql(left_sql, f"{df_concat_with_tf_table_name}_left")
+        pipeline.enqueue_sql(right_sql, f"{df_concat_with_tf_table_name}_right")
 
         blocking_input_tablename_l = f"{df_concat_with_tf_table_name}_left"
         blocking_input_tablename_r = f"{df_concat_with_tf_table_name}_right"
@@ -704,7 +772,7 @@ def block_using_rules_sqls(
         )
         br_sqls.append(sql)
 
-    sql = " UNION ALL ".join(br_sqls)
+    sql = join_sql_with_union_all(br_sqls)
 
     if any(isinstance(br, ExplodingBlockingRule) for br in blocking_rules):
         sqls.append(