splink 5.0.0.dev2__tar.gz → 5.0.0.dev4__tar.gz

{splink-5.0.0.dev2 → splink-5.0.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splink
-Version: 5.0.0.dev2
+Version: 5.0.0.dev4
 Summary: Fast probabilistic data linkage at scale
 Project-URL: Homepage, https://github.com/moj-analytical-services/splink
 Project-URL: Repository, https://github.com/moj-analytical-services/splink
@@ -8,7 +8,7 @@ Author: Sam Lindsay, Theodore Manassis, Tom Hepworth, Andy Bond, Ross Kennedy
 Author-email: Robin Linacre <robinlinacre@hotmail.com>
 License-Expression: MIT
 License-File: LICENSE
-Requires-Python: <4.0.0,>=3.9.0
+Requires-Python: <4.0.0,>=3.10.0
 Requires-Dist: altair>=5.0.1
 Requires-Dist: duckdb>=0.9.2
 Requires-Dist: igraph>=0.11.2
@@ -60,15 +60,15 @@ Splink's linkage algorithm is based on Fellegi-Sunter's model of record linkage,
 
 Consider the following records that lack a unique person identifier:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_1.drawio.png)
+![Input records that lack a unique person identifier](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_01_input_records.png)
 
 Splink predicts which rows link together:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_2.drawio.png)
+![Pairwise predictions with match probabilities](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_02_pairwise_links.png)
 
 and clusters these links to produce an estimated person ID:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_3.drawio.png)
+![Clusters of linked records forming estimated person IDs](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_03_clusters.png)
 
 ## What data does Splink work best with?
 

{splink-5.0.0.dev2 → splink-5.0.0.dev4}/README.md

@@ -31,15 +31,15 @@ Splink's linkage algorithm is based on Fellegi-Sunter's model of record linkage,
 
 Consider the following records that lack a unique person identifier:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_1.drawio.png)
+![Input records that lack a unique person identifier](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_01_input_records.png)
 
 Splink predicts which rows link together:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_2.drawio.png)
+![Pairwise predictions with match probabilities](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_02_pairwise_links.png)
 
 and clusters these links to produce an estimated person ID:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_3.drawio.png)
+![Clusters of linked records forming estimated person IDs](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_03_clusters.png)
 
 ## What data does Splink work best with?
 

{splink-5.0.0.dev2 → splink-5.0.0.dev4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splink"
-version = "5.0.0.dev2"
+version = "5.0.0.dev4"
 description = "Fast probabilistic data linkage at scale"
 authors = [
     { name = "Robin Linacre", email = "robinlinacre@hotmail.com" },
@@ -10,7 +10,7 @@ authors = [
     { name = "Andy Bond" },
     { name = "Ross Kennedy" },
 ]
-requires-python = ">=3.9.0,<4.0.0"
+requires-python = ">=3.10.0,<4.0.0"
 readme = "README.md"
 license = "MIT"
 dependencies = [
@@ -44,6 +44,7 @@ demos = [
     "nbmake>=1.5.5",
     "pseudopeople>=1.2.6",
     "pytest-xdist>=3.6.1",
+    { "include-group" = "pandas" },
 ]
 # dev deliberately doesn't include spark, so we can opt in to different spark versions
 dev = [
@@ -60,6 +61,7 @@ docs = [
     # keep jinja2 relaxed so it doesn't constrain us in rest of dev
     "Jinja2>=3.1.6",
     "mkdocs>=1.6.1",
+    "mkdocs-llmstxt>=0.5.0",
     # TODO: maybe we can use this to remove generating dataset docs as separate step
     #"mkdocs-gen-files==0.5.0",
     "mkdocs-material>=9.6.22",
@@ -72,7 +74,7 @@ docs = [
     "neoteroi-mkdocs>=1.1.2",
     "pymdown-extensions>=10.15",
 ]
-linting = ["ruff>=0.4.2,<0.5"]
+linting = ["ruff>=0.15.18"]
 pandas = ["pandas>=1.3.5", "numpy>=1.19.3"]
 postgres = [
     "psycopg2-binary>=2.9.0",
@@ -86,22 +88,22 @@ testing-core = [
     "pyarrow>=15.0.0",
     "networkx>=2.5.1",
     "pytest-cov>=5.0.0",
+    "pytest-xdist>=3.0.0",
     { "include-group" = "postgres" },
     { "include-group" = "sqlite" },
-    # TODO: temporary measure until we can depanda tests
-    { "include-group" = "pandas" },
 ]
 testing = [
     { "include-group" = "testing-core" },
     { "include-group" = "spark-4" },
 ]
-typechecking = ["mypy==1.9.0"]
+typechecking = ["mypy>=2.1.0"]
 # we will migrate to ty, but won't happen all at once, so run in parallel
 typechecking-dev = [
     "ty>=0.0.21",
 ]
 
 [tool.uv]
+exclude-newer = "7 days"
 default-groups = [
     "dev",
     "spark-4"

{splink-5.0.0.dev2 → splink-5.0.0.dev4}/splink/__init__.py

@@ -56,7 +56,7 @@ def __getattr__(name):
     raise AttributeError(f"module 'splink' has no attribute '{name}'") from None
 
 
-__version__ = "5.0.0.dev2"
+__version__ = "5.0.0.dev4"
 
 
 __all__ = [

splink-5.0.0.dev4/splink/backends/duckdb.py

@@ -0,0 +1,6 @@
+from splink.internals.duckdb.database_api import DuckDBAPI
+from splink.internals.duckdb.database_api_with_profiling import (
+    DuckDBAPIWithProfiling,
+)
+
+__all__ = ["DuckDBAPI", "DuckDBAPIWithProfiling"]

splink-5.0.0.dev4/splink/backends/spark.py

@@ -0,0 +1,5 @@
+from splink.internals.spark.database_api import SparkAPI
+from splink.internals.spark.database_api_with_profiling import SparkAPIWithProfiling
+from splink.internals.spark.jar_location import similarity_jar_location
+
+__all__ = ["similarity_jar_location", "SparkAPI", "SparkAPIWithProfiling"]

splink-5.0.0.dev4/splink/blocking_analysis.py

@@ -0,0 +1,11 @@
+from .internals.blocking_analysis import (
+    chart_comparisons_from_blocking_rules,
+    count_comparisons_from_blocking_rules,
+    n_largest_blocks,
+)
+
+__all__ = [
+    "count_comparisons_from_blocking_rules",
+    "chart_comparisons_from_blocking_rules",
+    "n_largest_blocks",
+]

splink-5.0.0.dev4/splink/exploratory.py

@@ -0,0 +1,4 @@
+from .internals.completeness import completeness_chart
+from .internals.profile_data import profile_columns
+
+__all__ = ["completeness_chart", "profile_columns"]

{splink-5.0.0.dev2 → splink-5.0.0.dev4}/splink/internals/accuracy.py

@@ -14,8 +14,8 @@ from splink.internals.predict import predict_from_comparison_vectors_sqls_using_
 from splink.internals.splink_dataframe import SplinkDataFrame
 from splink.internals.sql_transform import move_l_r_table_prefix_to_column_suffix
 from splink.internals.vertically_concatenate import (
-    compute_df_concat,
-    compute_df_concat_with_tf,
+    enqueue_df_concat,
+    enqueue_df_concat_with_tf,
 )
 
 if TYPE_CHECKING:
@@ -316,9 +316,7 @@ def truth_space_table_from_labels_table(
     match_weight_round_to_nearest: Optional[float] = None,
 ) -> SplinkDataFrame:
     pipeline = CTEPipeline()
-
-    nodes_with_tf = compute_df_concat_with_tf(linker, pipeline)
-    pipeline = CTEPipeline([nodes_with_tf])
+    enqueue_df_concat_with_tf(linker, pipeline)
 
     sqls = predictions_from_sample_of_pairwise_labels_sql(linker, labels_tablename)
     pipeline.enqueue_list_of_sqls(sqls)
@@ -351,19 +349,17 @@ def truth_space_table_from_labels_column(
         group_by_statement = "group by source_dataset"
 
     pipeline = CTEPipeline()
-    concat = compute_df_concat(linker, pipeline)
-
-    pipeline = CTEPipeline([concat])
+    enqueue_df_concat(linker, pipeline)
 
     sql = f"""
         select count(*) as count
-        from {concat.physical_name}
+        from __splink__df_concat
         {group_by_statement}
     """
 
     pipeline.enqueue_sql(sql, "__splink__cartesian_product")
     cartesian_count = linker._db_api.sql_pipeline_to_splink_dataframe(pipeline)
-    row_count_df = cartesian_count.as_record_dict()
+    row_count_df = cartesian_count.as_record_list()
     cartesian_count.drop_table_from_database_and_remove_from_cache()
 
     total_labels = calculate_cartesian(row_count_df, link_type)
@@ -451,8 +447,7 @@ def prediction_errors_from_labels_table(
     threshold_match_probability: float = 0.5,
 ) -> SplinkDataFrame:
     pipeline = CTEPipeline()
-    nodes_with_tf = compute_df_concat_with_tf(linker, pipeline)
-    pipeline = CTEPipeline([nodes_with_tf])
+    enqueue_df_concat_with_tf(linker, pipeline)
 
     sqls = predictions_from_sample_of_pairwise_labels_sql(linker, labels_tablename)
 

{splink-5.0.0.dev2 → splink-5.0.0.dev4}/splink/internals/blocking.py

@@ -5,16 +5,23 @@ import time
 from typing import TYPE_CHECKING, Any, List, Literal, Optional, TypedDict
 
 from sqlglot import parse_one
-from sqlglot.expressions import Column, Expression, Identifier, Join
+from sqlglot.expressions import Column, Identifier, Join
 from sqlglot.optimizer.eliminate_joins import join_condition
 from sqlglot.optimizer.optimizer import optimize
 from sqlglot.optimizer.simplify import flatten
 
-from splink.internals.chunking import _chunk_assignment_sql
+from splink.internals.chunking import (
+    _chunk_assignment_sql,
+)
 from splink.internals.database_api import DatabaseAPISubClass
 from splink.internals.dialects import SplinkDialect
+from splink.internals.em_sampling import _em_sample_filter_sql
 from splink.internals.input_column import InputColumn
-from splink.internals.misc import dedupe_preserving_order, ensure_is_list
+from splink.internals.misc import (
+    dedupe_preserving_order,
+    ensure_is_list,
+    join_sql_with_union_all,
+)
 from splink.internals.parse_sql import parse_columns_in_sql
 from splink.internals.pipeline import CTEPipeline
 from splink.internals.splink_dataframe import SplinkDataFrame
@@ -28,6 +35,9 @@ logger = logging.getLogger(__name__)
 
 # https://stackoverflow.com/questions/39740632/python-type-hinting-without-cyclic-imports
 if TYPE_CHECKING:
+    # Expr only exists in sqlglot >= 30.0.0
+    from sqlglot import Expr
+
     from splink.internals.settings import LinkTypeLiteralType
 
 user_input_link_type_options = Literal["link_only", "link_and_dedupe", "dedupe_only"]
@@ -117,6 +127,10 @@ class BlockingRule:
     def sqlglot_dialect(self) -> str:
         return SplinkDialect.from_string(self._sql_dialect_str).sqlglot_dialect
 
+    @property
+    def requires_blocking_input_materialisation(self) -> bool:
+        return False
+
     @property
     def sql_dialect(self) -> SplinkDialect:
         return SplinkDialect.from_string(self._sql_dialect_str)
@@ -203,10 +217,11 @@ class BlockingRule:
             on
             ({self.blocking_rule_sql})
             {where_condition}
-            {self.exclude_pairs_generated_by_all_preceding_rules_sql(
-                source_dataset_input_column,
-                unique_id_input_column)
-            }
+            {
+            self.exclude_pairs_generated_by_all_preceding_rules_sql(
+                source_dataset_input_column, unique_id_input_column
+            )
+        }
             """
         return sql
 
@@ -239,7 +254,7 @@ class BlockingRule:
             list of tuples like [(name, name), (substr(name,1,2), substr(name,2,3))]
         """
 
-        def remove_table_prefix(tree: Expression) -> Expression:
+        def remove_table_prefix(tree: Expr) -> Expr:
             for c in tree.find_all(Column):
                 del c.args["table"]
             return tree
@@ -252,7 +267,7 @@ class BlockingRule:
 
         rmtp = remove_table_prefix
 
-        keys_de_prefixed: list[tuple[Expression, Expression]] = [
+        keys_de_prefixed: list[tuple[Expr, Expr]] = [
             (rmtp(i), rmtp(j)) for (i, j) in keys_zipped
         ]
 
@@ -342,6 +357,8 @@ class ExplodingBlockingRule(BlockingRule):
         input_tablename_r: str,
         left_chunk: tuple[int, int] | None = None,
         right_chunk: tuple[int, int] | None = None,
+        sample_threshold: int | None = None,
+        sample_modulus: int | None = None,
     ) -> str:
         """generates a table of the marginal id pairs from the exploded blocking rule
         i.e. pairs are only created that match this blocking rule and NOT any of
@@ -352,6 +369,9 @@ class ExplodingBlockingRule(BlockingRule):
                 left side records.
             right_chunk: Optional tuple of (chunk_number, total_chunks) for filtering
                 right side records.
+            sample_threshold: Optional integer threshold for deterministic hash
+                sampling of both sides of the join (used by blocking analysis).
+            sample_modulus: Optional modulus for the deterministic hash sample.
         """
 
         unique_id_col = unique_id_input_column
@@ -365,6 +385,8 @@ class ExplodingBlockingRule(BlockingRule):
             left_chunk=left_chunk,
             right_chunk=right_chunk,
             sql_dialect=self.sql_dialect,
+            sample_threshold=sample_threshold,
+            sample_modulus=sample_modulus,
         )
 
         id_expr_l = _composite_unique_id_from_nodes_sql(unique_id_input_columns, "l")
@@ -403,6 +425,10 @@ class ExplodingBlockingRule(BlockingRule):
 
         return "false"
 
+    @property
+    def requires_blocking_input_materialisation(self) -> bool:
+        return True
+
     def create_blocked_pairs_sql(
         self,
         *,
@@ -467,6 +493,8 @@ def materialise_exploded_id_tables(
     unique_id_input_column: InputColumn,
     left_chunk: tuple[int, int] | None = None,
     right_chunk: tuple[int, int] | None = None,
+    sample_threshold: int | None = None,
+    sample_modulus: int | None = None,
 ) -> list[ExplodingBlockingRule]:
     """Materialise exploded ID pair tables for exploding blocking rules.
 
@@ -475,6 +503,9 @@ def materialise_exploded_id_tables(
             left side records.
         right_chunk: Optional tuple of (chunk_number, total_chunks) for filtering
             right side records.
+        sample_threshold: Optional integer threshold for deterministic hash
+            sampling of both sides of the join (used by blocking analysis).
+        sample_modulus: Optional modulus for the deterministic hash sample.
     """
     exploding_blocking_rules = [
         br for br in blocking_rules if isinstance(br, ExplodingBlockingRule)
@@ -556,6 +587,8 @@ def materialise_exploded_id_tables(
             input_tablename_r=input_tablename_r,
             left_chunk=left_chunk,
             right_chunk=right_chunk,
+            sample_threshold=sample_threshold,
+            sample_modulus=sample_modulus,
         )
 
         pipeline.enqueue_sql(sql, table_name)
@@ -668,6 +701,8 @@ def _sql_gen_where_condition(
     left_chunk: tuple[int, int] | None = None,
     right_chunk: tuple[int, int] | None = None,
     sql_dialect: "SplinkDialect | None" = None,
+    sample_threshold: int | None = None,
+    sample_modulus: int | None = None,
 ) -> str:
     id_expr_l = _composite_unique_id_from_nodes_sql(unique_id_cols, "l")
     id_expr_r = _composite_unique_id_from_nodes_sql(unique_id_cols, "r")
@@ -696,6 +731,16 @@ def _sql_gen_where_condition(
             unique_id_cols, chunk_num, total_chunks, "r", sql_dialect
         )
 
+    if sample_threshold is not None and sample_modulus is not None:
+        if sql_dialect is None:
+            raise ValueError("EM sampling requires a SQL dialect")
+        where_condition += _em_sample_filter_sql(
+            unique_id_cols, sample_threshold, sample_modulus, "l", sql_dialect
+        )
+        where_condition += _em_sample_filter_sql(
+            unique_id_cols, sample_threshold, sample_modulus, "r", sql_dialect
+        )
+
     return where_condition
 
 
@@ -709,6 +754,8 @@ def block_using_rules_sqls(
     unique_id_input_column: InputColumn,
     left_chunk: tuple[int, int] | None = None,
     right_chunk: tuple[int, int] | None = None,
+    sample_threshold: int | None = None,
+    sample_modulus: int | None = None,
 ) -> list[dict[str, str]]:
     """Use the blocking rules specified in the linker's settings object to
     generate a SQL statement that will create pairwise record comparions
@@ -739,6 +786,8 @@ def block_using_rules_sqls(
         left_chunk=left_chunk,
         right_chunk=right_chunk,
         sql_dialect=sql_dialect,
+        sample_threshold=sample_threshold,
+        sample_modulus=sample_modulus,
     )
 
     # Cover the case where there are no blocking rules
@@ -760,7 +809,7 @@ def block_using_rules_sqls(
         )
         br_sqls.append(sql)
 
-    sql = " UNION ALL ".join(br_sqls)
+    sql = join_sql_with_union_all(br_sqls)
 
     if any(isinstance(br, ExplodingBlockingRule) for br in blocking_rules):
         sqls.append(