splink 5.0.0.dev3__tar.gz → 5.0.0.dev5__tar.gz

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/.gitignore

@@ -169,7 +169,12 @@ cython_debug/
 # html files opt-in only
 *.html
 !splink/**/templates/**/*.html
-!docs/**/*.html
+!docs/demos/examples/**/*.html
+!docs/demos/tutorials/**/*.html
+!docs/overrides/**/*.html
+
+# don't store ipynb here
+*.ipynb
 
 .DS_Store
 

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: splink
-Version: 5.0.0.dev3
+Version: 5.0.0.dev5
 Summary: Fast probabilistic data linkage at scale
 Project-URL: Homepage, https://github.com/moj-analytical-services/splink
 Project-URL: Repository, https://github.com/moj-analytical-services/splink
@@ -8,12 +8,13 @@ Author: Sam Lindsay, Theodore Manassis, Tom Hepworth, Andy Bond, Ross Kennedy
 Author-email: Robin Linacre <robinlinacre@hotmail.com>
 License-Expression: MIT
 License-File: LICENSE
-Requires-Python: <4.0.0,>=3.9.0
-Requires-Dist: altair>=5.0.1
+Requires-Python: <4.0.0,>=3.10.0
 Requires-Dist: duckdb>=0.9.2
-Requires-Dist: igraph>=0.11.2
-Requires-Dist: jinja2>=3.0.3
 Requires-Dist: sqlglot>=17.6.0
+Provides-Extra: altair
+Requires-Dist: altair>=5.0.1; extra == 'altair'
+Provides-Extra: igraph
+Requires-Dist: igraph>=0.11.2; extra == 'igraph'
 Provides-Extra: postgres
 Requires-Dist: psycopg2-binary>=2.9.0; extra == 'postgres'
 Requires-Dist: sqlalchemy>=2.0.0; extra == 'postgres'
@@ -60,15 +61,15 @@ Splink's linkage algorithm is based on Fellegi-Sunter's model of record linkage,
 
 Consider the following records that lack a unique person identifier:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_1.drawio.png)
+![Input records that lack a unique person identifier](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_01_input_records.png)
 
 Splink predicts which rows link together:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_2.drawio.png)
+![Pairwise predictions with match probabilities](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_02_pairwise_links.png)
 
 and clusters these links to produce an estimated person ID:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_3.drawio.png)
+![Clusters of linked records forming estimated person IDs](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_03_clusters.png)
 
 ## What data does Splink work best with?
 

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/README.md

@@ -31,15 +31,15 @@ Splink's linkage algorithm is based on Fellegi-Sunter's model of record linkage,
 
 Consider the following records that lack a unique person identifier:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_1.drawio.png)
+![Input records that lack a unique person identifier](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_01_input_records.png)
 
 Splink predicts which rows link together:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_2.drawio.png)
+![Pairwise predictions with match probabilities](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_02_pairwise_links.png)
 
 and clusters these links to produce an estimated person ID:
 
-![tables showing what splink does](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/what_does_splink_do_3.drawio.png)
+![Clusters of linked records forming estimated person IDs](https://raw.githubusercontent.com/moj-analytical-services/splink/master/docs/img/README/splink_03_clusters.png)
 
 ## What data does Splink work best with?
 

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splink"
-version = "5.0.0.dev3"
+version = "5.0.0.dev5"
 description = "Fast probabilistic data linkage at scale"
 authors = [
     { name = "Robin Linacre", email = "robinlinacre@hotmail.com" },
@@ -10,18 +10,17 @@ authors = [
     { name = "Andy Bond" },
     { name = "Ross Kennedy" },
 ]
-requires-python = ">=3.9.0,<4.0.0"
+requires-python = ">=3.10.0,<4.0.0"
 readme = "README.md"
 license = "MIT"
 dependencies = [
     "duckdb>=0.9.2",
     "sqlglot>=17.6.0",
-    "altair>=5.0.1",
-    "Jinja2>=3.0.3",
-    "igraph>=0.11.2",
 ]
 
 [project.optional-dependencies]
+altair = ["altair>=5.0.1"]
+igraph = ["igraph>=0.11.2"]
 pyspark = ["pyspark>=3.5.0"]
 spark = ["pyspark>=3.5.0"]
 # sqlite doesn't _require_ this functionality, but it's used if we want to use fuzzy levels
@@ -38,13 +37,6 @@ Repository = "https://github.com/moj-analytical-services/splink"
 # allows us to easily check package functionality with just these installed
 # (and not the rest of our required dependencies)
 core = ["sqlglot", "duckdb"]
-demos = [
-    "ipywidgets>=8.1.7",
-    "jupytext>=1.18.1",
-    "nbmake>=1.5.5",
-    "pseudopeople>=1.2.6",
-    "pytest-xdist>=3.6.1",
-]
 # dev deliberately doesn't include spark, so we can opt in to different spark versions
 dev = [
     { "include-group" = "linting" },
@@ -56,10 +48,12 @@ spark-3 = [
 ]
 spark-4 = ["pyspark>=4.0.0"]
 docs = [
+    "altair>=5.0.1",
     "griffe>=1.4.0",
     # keep jinja2 relaxed so it doesn't constrain us in rest of dev
     "Jinja2>=3.1.6",
     "mkdocs>=1.6.1",
+    "mkdocs-llmstxt>=0.5.0",
     # TODO: maybe we can use this to remove generating dataset docs as separate step
     #"mkdocs-gen-files==0.5.0",
     "mkdocs-material>=9.6.22",
@@ -72,7 +66,7 @@ docs = [
     "neoteroi-mkdocs>=1.1.2",
     "pymdown-extensions>=10.15",
 ]
-linting = ["ruff>=0.4.2,<0.5"]
+linting = ["ruff>=0.15.18"]
 pandas = ["pandas>=1.3.5", "numpy>=1.19.3"]
 postgres = [
     "psycopg2-binary>=2.9.0",
@@ -82,26 +76,27 @@ sqlite = [
     "rapidfuzz>=3.10.0",
 ]
 testing-core = [
+    "igraph>=0.11.2",
     "pytest>=7.3",
     "pyarrow>=15.0.0",
     "networkx>=2.5.1",
     "pytest-cov>=5.0.0",
+    "pytest-xdist>=3.0.0",
     { "include-group" = "postgres" },
     { "include-group" = "sqlite" },
-    # TODO: temporary measure until we can depanda tests
-    { "include-group" = "pandas" },
 ]
 testing = [
     { "include-group" = "testing-core" },
     { "include-group" = "spark-4" },
 ]
-typechecking = ["mypy==1.9.0"]
+typechecking = ["mypy>=2.1.0"]
 # we will migrate to ty, but won't happen all at once, so run in parallel
 typechecking-dev = [
     "ty>=0.0.21",
 ]
 
 [tool.uv]
+exclude-newer = "7 days"
 default-groups = [
     "dev",
     "spark-4"
@@ -179,9 +174,6 @@ markers = [
 [tool.mypy]
 python_version = "3.10"
 packages = "splink"
-# for now at least allow implicit optionals
-# to cut down on noise. Easy to fix.
-implicit_optional = true
 # for now, ignore missing imports
 # can remove later and install stubs, where existent
 ignore_missing_imports = true
@@ -203,3 +195,6 @@ disallow_any_generics = true
 # disallow_untyped_calls = true
 disallow_incomplete_defs = true
 # disallow_untyped_defs = true
+
+[tool.ty.rules]
+unresolved-import = "ignore"

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/splink/__init__.py

@@ -56,7 +56,7 @@ def __getattr__(name):
     raise AttributeError(f"module 'splink' has no attribute '{name}'") from None
 
 
-__version__ = "5.0.0.dev3"
+__version__ = "5.0.0.dev5"
 
 
 __all__ = [

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/splink/backends/duckdb.py

@@ -2,5 +2,6 @@ from splink.internals.duckdb.database_api import DuckDBAPI
 from splink.internals.duckdb.database_api_with_profiling import (
     DuckDBAPIWithProfiling,
 )
+from splink.internals.duckdb.parquet_write_options import ParquetWriteOptions
 
-__all__ = ["DuckDBAPI", "DuckDBAPIWithProfiling"]
+__all__ = ["DuckDBAPI", "DuckDBAPIWithProfiling", "ParquetWriteOptions"]

splink-5.0.0.dev5/splink/blocking_analysis.py

@@ -0,0 +1,11 @@
+from .internals.blocking_analysis import (
+    chart_comparisons_from_blocking_rules,
+    count_comparisons_from_blocking_rules,
+    n_largest_blocks,
+)
+
+__all__ = [
+    "count_comparisons_from_blocking_rules",
+    "chart_comparisons_from_blocking_rules",
+    "n_largest_blocks",
+]

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/splink/internals/accuracy.py

@@ -26,7 +26,7 @@ if TYPE_CHECKING:
 def truth_space_table_from_labels_with_predictions_sqls(
     threshold_actual: float = 0.5,
     match_weight_round_to_nearest: float | None = None,
-    total_labels: int = None,
+    total_labels: int | None = None,
     positives_not_captured_by_blocking_rules_scored_as_zero: bool = True,
 ) -> list[dict[str, str]]:
     """
@@ -336,7 +336,7 @@ def truth_space_table_from_labels_column(
     linker: "Linker",
     label_colname: str,
     threshold_actual: float = 0.5,
-    match_weight_round_to_nearest: float = None,
+    match_weight_round_to_nearest: float | None = None,
     positives_not_captured_by_blocking_rules_scored_as_zero: bool = True,
 ) -> SplinkDataFrame:
     # First we need to calculate the number of implicit true negatives
@@ -359,7 +359,7 @@ def truth_space_table_from_labels_column(
 
     pipeline.enqueue_sql(sql, "__splink__cartesian_product")
     cartesian_count = linker._db_api.sql_pipeline_to_splink_dataframe(pipeline)
-    row_count_df = cartesian_count.as_record_dict()
+    row_count_df = cartesian_count.as_record_list()
     cartesian_count.drop_table_from_database_and_remove_from_cache()
 
     total_labels = calculate_cartesian(row_count_df, link_type)

{splink-5.0.0.dev3 → splink-5.0.0.dev5}/splink/internals/blocking.py

@@ -5,14 +5,17 @@ import time
 from typing import TYPE_CHECKING, Any, List, Literal, Optional, TypedDict
 
 from sqlglot import parse_one
-from sqlglot.expressions import Column, Expression, Identifier, Join
+from sqlglot.expressions import Column, Identifier, Join
 from sqlglot.optimizer.eliminate_joins import join_condition
 from sqlglot.optimizer.optimizer import optimize
 from sqlglot.optimizer.simplify import flatten
 
-from splink.internals.chunking import _chunk_assignment_sql
+from splink.internals.chunking import (
+    _chunk_assignment_sql,
+)
 from splink.internals.database_api import DatabaseAPISubClass
 from splink.internals.dialects import SplinkDialect
+from splink.internals.em_sampling import _em_sample_filter_sql
 from splink.internals.input_column import InputColumn
 from splink.internals.misc import (
     dedupe_preserving_order,
@@ -32,6 +35,9 @@ logger = logging.getLogger(__name__)
 
 # https://stackoverflow.com/questions/39740632/python-type-hinting-without-cyclic-imports
 if TYPE_CHECKING:
+    # Expr only exists in sqlglot >= 30.0.0
+    from sqlglot import Expr
+
     from splink.internals.settings import LinkTypeLiteralType
 
 user_input_link_type_options = Literal["link_only", "link_and_dedupe", "dedupe_only"]
@@ -211,10 +217,11 @@ class BlockingRule:
             on
             ({self.blocking_rule_sql})
             {where_condition}
-            {self.exclude_pairs_generated_by_all_preceding_rules_sql(
-                source_dataset_input_column,
-                unique_id_input_column)
-            }
+            {
+            self.exclude_pairs_generated_by_all_preceding_rules_sql(
+                source_dataset_input_column, unique_id_input_column
+            )
+        }
             """
         return sql
 
@@ -247,7 +254,7 @@ class BlockingRule:
             list of tuples like [(name, name), (substr(name,1,2), substr(name,2,3))]
         """
 
-        def remove_table_prefix(tree: Expression) -> Expression:
+        def remove_table_prefix(tree: Expr) -> Expr:
             for c in tree.find_all(Column):
                 del c.args["table"]
             return tree
@@ -260,7 +267,7 @@ class BlockingRule:
 
         rmtp = remove_table_prefix
 
-        keys_de_prefixed: list[tuple[Expression, Expression]] = [
+        keys_de_prefixed: list[tuple[Expr, Expr]] = [
             (rmtp(i), rmtp(j)) for (i, j) in keys_zipped
         ]
 
@@ -350,6 +357,8 @@ class ExplodingBlockingRule(BlockingRule):
         input_tablename_r: str,
         left_chunk: tuple[int, int] | None = None,
         right_chunk: tuple[int, int] | None = None,
+        sample_threshold: int | None = None,
+        sample_modulus: int | None = None,
     ) -> str:
         """generates a table of the marginal id pairs from the exploded blocking rule
         i.e. pairs are only created that match this blocking rule and NOT any of
@@ -360,6 +369,9 @@ class ExplodingBlockingRule(BlockingRule):
                 left side records.
             right_chunk: Optional tuple of (chunk_number, total_chunks) for filtering
                 right side records.
+            sample_threshold: Optional integer threshold for deterministic hash
+                sampling of both sides of the join (used by blocking analysis).
+            sample_modulus: Optional modulus for the deterministic hash sample.
         """
 
         unique_id_col = unique_id_input_column
@@ -373,6 +385,8 @@ class ExplodingBlockingRule(BlockingRule):
             left_chunk=left_chunk,
             right_chunk=right_chunk,
             sql_dialect=self.sql_dialect,
+            sample_threshold=sample_threshold,
+            sample_modulus=sample_modulus,
         )
 
         id_expr_l = _composite_unique_id_from_nodes_sql(unique_id_input_columns, "l")
@@ -479,6 +493,8 @@ def materialise_exploded_id_tables(
     unique_id_input_column: InputColumn,
     left_chunk: tuple[int, int] | None = None,
     right_chunk: tuple[int, int] | None = None,
+    sample_threshold: int | None = None,
+    sample_modulus: int | None = None,
 ) -> list[ExplodingBlockingRule]:
     """Materialise exploded ID pair tables for exploding blocking rules.
 
@@ -487,6 +503,9 @@ def materialise_exploded_id_tables(
             left side records.
         right_chunk: Optional tuple of (chunk_number, total_chunks) for filtering
             right side records.
+        sample_threshold: Optional integer threshold for deterministic hash
+            sampling of both sides of the join (used by blocking analysis).
+        sample_modulus: Optional modulus for the deterministic hash sample.
     """
     exploding_blocking_rules = [
         br for br in blocking_rules if isinstance(br, ExplodingBlockingRule)
@@ -568,6 +587,8 @@ def materialise_exploded_id_tables(
             input_tablename_r=input_tablename_r,
             left_chunk=left_chunk,
             right_chunk=right_chunk,
+            sample_threshold=sample_threshold,
+            sample_modulus=sample_modulus,
         )
 
         pipeline.enqueue_sql(sql, table_name)
@@ -680,6 +701,8 @@ def _sql_gen_where_condition(
     left_chunk: tuple[int, int] | None = None,
     right_chunk: tuple[int, int] | None = None,
     sql_dialect: "SplinkDialect | None" = None,
+    sample_threshold: int | None = None,
+    sample_modulus: int | None = None,
 ) -> str:
     id_expr_l = _composite_unique_id_from_nodes_sql(unique_id_cols, "l")
     id_expr_r = _composite_unique_id_from_nodes_sql(unique_id_cols, "r")
@@ -708,6 +731,16 @@ def _sql_gen_where_condition(
             unique_id_cols, chunk_num, total_chunks, "r", sql_dialect
         )
 
+    if sample_threshold is not None and sample_modulus is not None:
+        if sql_dialect is None:
+            raise ValueError("EM sampling requires a SQL dialect")
+        where_condition += _em_sample_filter_sql(
+            unique_id_cols, sample_threshold, sample_modulus, "l", sql_dialect
+        )
+        where_condition += _em_sample_filter_sql(
+            unique_id_cols, sample_threshold, sample_modulus, "r", sql_dialect
+        )
+
     return where_condition
 
 
@@ -721,6 +754,8 @@ def block_using_rules_sqls(
     unique_id_input_column: InputColumn,
     left_chunk: tuple[int, int] | None = None,
     right_chunk: tuple[int, int] | None = None,
+    sample_threshold: int | None = None,
+    sample_modulus: int | None = None,
 ) -> list[dict[str, str]]:
     """Use the blocking rules specified in the linker's settings object to
     generate a SQL statement that will create pairwise record comparions
@@ -751,6 +786,8 @@ def block_using_rules_sqls(
         left_chunk=left_chunk,
         right_chunk=right_chunk,
         sql_dialect=sql_dialect,
+        sample_threshold=sample_threshold,
+        sample_modulus=sample_modulus,
     )
 
     # Cover the case where there are no blocking rules