splink 5.0.0.dev2__tar.gz → 5.0.0.dev3__tar.gz

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splink
-Version: 5.0.0.dev2
+Version: 5.0.0.dev3
 Summary: Fast probabilistic data linkage at scale
 Project-URL: Homepage, https://github.com/moj-analytical-services/splink
 Project-URL: Repository, https://github.com/moj-analytical-services/splink

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "splink"
-version = "5.0.0.dev2"
+version = "5.0.0.dev3"
 description = "Fast probabilistic data linkage at scale"
 authors = [
     { name = "Robin Linacre", email = "robinlinacre@hotmail.com" },

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/__init__.py

@@ -56,7 +56,7 @@ def __getattr__(name):
     raise AttributeError(f"module 'splink' has no attribute '{name}'") from None
 
 
-__version__ = "5.0.0.dev2"
+__version__ = "5.0.0.dev3"
 
 
 __all__ = [

splink-5.0.0.dev3/splink/backends/duckdb.py

@@ -0,0 +1,6 @@
+from splink.internals.duckdb.database_api import DuckDBAPI
+from splink.internals.duckdb.database_api_with_profiling import (
+    DuckDBAPIWithProfiling,
+)
+
+__all__ = ["DuckDBAPI", "DuckDBAPIWithProfiling"]

splink-5.0.0.dev3/splink/backends/spark.py

@@ -0,0 +1,5 @@
+from splink.internals.spark.database_api import SparkAPI
+from splink.internals.spark.database_api_with_profiling import SparkAPIWithProfiling
+from splink.internals.spark.jar_location import similarity_jar_location
+
+__all__ = ["similarity_jar_location", "SparkAPI", "SparkAPIWithProfiling"]

splink-5.0.0.dev3/splink/exploratory.py

@@ -0,0 +1,4 @@
+from .internals.completeness import completeness_chart
+from .internals.profile_data import profile_columns
+
+__all__ = ["completeness_chart", "profile_columns"]

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/accuracy.py

@@ -14,8 +14,8 @@ from splink.internals.predict import predict_from_comparison_vectors_sqls_using_
 from splink.internals.splink_dataframe import SplinkDataFrame
 from splink.internals.sql_transform import move_l_r_table_prefix_to_column_suffix
 from splink.internals.vertically_concatenate import (
-    compute_df_concat,
-    compute_df_concat_with_tf,
+    enqueue_df_concat,
+    enqueue_df_concat_with_tf,
 )
 
 if TYPE_CHECKING:
@@ -316,9 +316,7 @@ def truth_space_table_from_labels_table(
     match_weight_round_to_nearest: Optional[float] = None,
 ) -> SplinkDataFrame:
     pipeline = CTEPipeline()
-
-    nodes_with_tf = compute_df_concat_with_tf(linker, pipeline)
-    pipeline = CTEPipeline([nodes_with_tf])
+    enqueue_df_concat_with_tf(linker, pipeline)
 
     sqls = predictions_from_sample_of_pairwise_labels_sql(linker, labels_tablename)
     pipeline.enqueue_list_of_sqls(sqls)
@@ -351,13 +349,11 @@ def truth_space_table_from_labels_column(
         group_by_statement = "group by source_dataset"
 
     pipeline = CTEPipeline()
-    concat = compute_df_concat(linker, pipeline)
-
-    pipeline = CTEPipeline([concat])
+    enqueue_df_concat(linker, pipeline)
 
     sql = f"""
         select count(*) as count
-        from {concat.physical_name}
+        from __splink__df_concat
         {group_by_statement}
     """
 
@@ -451,8 +447,7 @@ def prediction_errors_from_labels_table(
     threshold_match_probability: float = 0.5,
 ) -> SplinkDataFrame:
     pipeline = CTEPipeline()
-    nodes_with_tf = compute_df_concat_with_tf(linker, pipeline)
-    pipeline = CTEPipeline([nodes_with_tf])
+    enqueue_df_concat_with_tf(linker, pipeline)
 
     sqls = predictions_from_sample_of_pairwise_labels_sql(linker, labels_tablename)
 

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/blocking.py

@@ -14,7 +14,11 @@ from splink.internals.chunking import _chunk_assignment_sql
 from splink.internals.database_api import DatabaseAPISubClass
 from splink.internals.dialects import SplinkDialect
 from splink.internals.input_column import InputColumn
-from splink.internals.misc import dedupe_preserving_order, ensure_is_list
+from splink.internals.misc import (
+    dedupe_preserving_order,
+    ensure_is_list,
+    join_sql_with_union_all,
+)
 from splink.internals.parse_sql import parse_columns_in_sql
 from splink.internals.pipeline import CTEPipeline
 from splink.internals.splink_dataframe import SplinkDataFrame
@@ -117,6 +121,10 @@ class BlockingRule:
     def sqlglot_dialect(self) -> str:
         return SplinkDialect.from_string(self._sql_dialect_str).sqlglot_dialect
 
+    @property
+    def requires_blocking_input_materialisation(self) -> bool:
+        return False
+
     @property
     def sql_dialect(self) -> SplinkDialect:
         return SplinkDialect.from_string(self._sql_dialect_str)
@@ -403,6 +411,10 @@ class ExplodingBlockingRule(BlockingRule):
 
         return "false"
 
+    @property
+    def requires_blocking_input_materialisation(self) -> bool:
+        return True
+
     def create_blocked_pairs_sql(
         self,
         *,
@@ -760,7 +772,7 @@ def block_using_rules_sqls(
         )
         br_sqls.append(sql)
 
-    sql = " UNION ALL ".join(br_sqls)
+    sql = join_sql_with_union_all(br_sqls)
 
     if any(isinstance(br, ExplodingBlockingRule) for br in blocking_rules):
         sqls.append(

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/blocking_analysis.py

@@ -15,7 +15,6 @@ from typing import (
     cast,
 )
 
-import duckdb
 import sqlglot
 
 from splink.internals.blocking import (
@@ -77,22 +76,23 @@ def _count_comparisons_from_blocking_rule_pre_filter_conditions_sqls(
         input_tablename_l = "__splink__df_concat"
         input_tablename_r = "__splink__df_concat"
 
-    sql = blocking_rule.create_blocking_input_sql(
-        input_tablename=input_tablename_l,
-        input_columns=input_dataframes[0].columns,
-    )
+    if blocking_rule.requires_blocking_input_materialisation:
+        sql = blocking_rule.create_blocking_input_sql(
+            input_tablename=input_tablename_l,
+            input_columns=input_dataframes[0].columns,
+        )
 
-    sqls.append({"sql": sql, "output_table_name": "__splink__br_input_l"})
+        sqls.append({"sql": sql, "output_table_name": "__splink__br_input_l"})
 
-    sql = blocking_rule.create_blocking_input_sql(
-        input_tablename=input_tablename_r,
-        input_columns=input_dataframes[0].columns,
-    )
+        sql = blocking_rule.create_blocking_input_sql(
+            input_tablename=input_tablename_r,
+            input_columns=input_dataframes[0].columns,
+        )
 
-    sqls.append({"sql": sql, "output_table_name": "__splink__br_input_r"})
+        sqls.append({"sql": sql, "output_table_name": "__splink__br_input_r"})
 
-    input_tablename_l = "__splink__br_input_l"
-    input_tablename_r = "__splink__br_input_r"
+        input_tablename_l = "__splink__br_input_l"
+        input_tablename_r = "__splink__br_input_r"
 
     l_cols_sel = []
     r_cols_sel = []
@@ -380,18 +380,19 @@ def _cumulative_comparisons_to_be_scored_from_blocking_rules(
     """
     pipeline.enqueue_sql(sql, "__splink__df_count_cumulative_blocks")
 
-    # TODO: duckdb (if available) or arrow (if not)
-    result_df = db_api.sql_pipeline_to_splink_dataframe(pipeline).as_pandas_dataframe()
-    # TODO: resuse connexion if available - see similar code in EM training
-    con = duckdb.connect()
+    result_df = db_api.sql_pipeline_to_splink_dataframe(pipeline).as_duckdbpyrelation()
+    con = db_api.duckdb_con
+    con.register("result_df", result_df)
 
     # The above table won't include rules that have no matches
+    all_rules_table_name = "__splink__df_blocking_rule_counts"
     con.execute(
-        "CREATE TABLE all_rules "
-        "(match_key VARCHAR, blocking_rule VARCHAR, cartesian BIGINT);"
+        f"CREATE OR REPLACE TABLE {all_rules_table_name} "
+        "(match_key BIGINT, blocking_rule VARCHAR, cartesian BIGINT);"
     )
     con.executemany(
-        "INSERT INTO all_rules VALUES ($match_key, $blocking_rule, $cartesian);",
+        f"INSERT INTO {all_rules_table_name} VALUES "
+        "($match_key, $blocking_rule, $cartesian);",
         [
             {
                 "match_key": str(i),
@@ -402,8 +403,8 @@ def _cumulative_comparisons_to_be_scored_from_blocking_rules(
         ],
     )
 
+    table_name = "__splink__cumulative_blocking_rule_counts"
     if len(result_df) > 0:
-        table_name = "__splink__cumulative_blocking_rule_counts"
         con.register(table_name, result_df)
         sql = f"""
             WITH simple_counts AS (
@@ -413,7 +414,7 @@ def _cumulative_comparisons_to_be_scored_from_blocking_rules(
                     cast(rules.match_key as int) as match_key,
                     rules.cartesian
                 FROM
-                    all_rules AS rules
+                    {all_rules_table_name} AS rules
                 LEFT JOIN
                     {table_name} AS counts
                 ON
@@ -437,7 +438,7 @@ def _cumulative_comparisons_to_be_scored_from_blocking_rules(
     else:
         # TODO: can we join onto empty arrow table? if so, we don't need separate case
         # simpler sql as we have no data to join onto
-        sql = """
+        sql = f"""
             select
                 blocking_rule,
                 0 as row_count,
@@ -446,15 +447,18 @@ def _cumulative_comparisons_to_be_scored_from_blocking_rules(
                 match_key,
                 0 as start,
             from
-                all_rules
+                {all_rules_table_name}
         """
 
     [b.drop_materialised_id_pairs_dataframe() for b in exploding_br_with_id_tables]
     complete_df = con.sql(sql)
-
-    return cast(
+    counts_data = cast(
         list[CumulativeComparisonRecord], record_dicts_from_relation(complete_df)
     )
+    # clean up temporary tables
+    con.execute(f"DROP VIEW IF EXISTS {table_name}")
+    con.execute(f"DROP TABLE {all_rules_table_name}")
+    return counts_data
 
 
 def _count_comparisons_generated_from_blocking_rule(

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/clustering.py

@@ -9,6 +9,7 @@ from splink.internals.database_api import AcceptableInputTableType, DatabaseAPIS
 from splink.internals.input_column import InputColumn
 from splink.internals.misc import (
     ascii_uid,
+    join_sql_with_union_all,
     prob_to_match_weight,
     threshold_args_to_match_prob,
     threshold_args_to_match_prob_list,
@@ -338,7 +339,7 @@ def _generate_cluster_summary_stats_sql(
         for threshold in thresholds
     ]
 
-    sql = "\nUNION ALL\n".join(select_statements)
+    sql = join_sql_with_union_all(select_statements)
 
     return sql
 

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/comparison.py

@@ -7,6 +7,7 @@ from splink.internals.dialects import SplinkDialect
 from splink.internals.input_column import InputColumn
 from splink.internals.misc import (
     dedupe_preserving_order,
+    indent_sql,
     join_list_with_commas_final_and,
 )
 
@@ -161,8 +162,8 @@ class Comparison:
         sqls = [
             cl._when_then_comparison_vector_value_sql for cl in self.comparison_levels
         ]
-        sql = " ".join(sqls)
-        sql = f"CASE {sql} END as {self._gamma_column_name}"
+        sql = "\n".join(sqls)
+        sql = f"CASE\n{indent_sql(sql)}\nEND as {self._gamma_column_name}"
 
         return sql
 
@@ -253,8 +254,8 @@ class Comparison:
             cl._match_weight_sql(self._gamma_column_name)
             for cl in self.comparison_levels
         ]
-        sql = " ".join(sqls)
-        sql = f"CASE {sql} END as {self._mw_column_name}"
+        sql = "\n".join(sqls)
+        sql = f"CASE\n{indent_sql(sql)}\nEND as {self._mw_column_name}"
         output_cols.append(sql)
 
         if self._has_tf_adjustments:
@@ -262,8 +263,8 @@ class Comparison:
                 cl._tf_adjustment_sql(self._gamma_column_name, self.comparison_levels)
                 for cl in self.comparison_levels
             ]
-            sql = " ".join(sqls)
-            sql = f"\nCASE {sql} END as {self._mw_tf_adj_column_name}"
+            sql = "\n".join(sqls)
+            sql = f"CASE\n{indent_sql(sql)}\nEND as {self._mw_tf_adj_column_name}"
             output_cols.append(sql)
         output_cols.append(self._gamma_column_name)
 

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/comparison_level.py

@@ -10,7 +10,7 @@ from textwrap import dedent
 from typing import Any, Optional, Union, cast
 
 import sqlglot
-from sqlglot.expressions import Column, Identifier
+from sqlglot.expressions import Column, Expression, Identifier
 from sqlglot.optimizer.normalize import normalize
 from sqlglot.optimizer.simplify import simplify
 
@@ -84,12 +84,12 @@ def _exact_match_colname(sql_syntax_tree):
     return cols[0]
 
 
-def _get_and_subclauses(expr: sqlglot.Expression) -> list[sqlglot.Expression]:
+def _get_and_subclauses(expr: Expression) -> list[Expression]:
     # get list of subclauses joined together by 'AND' at top-level
     # e.g. 'A AND B AND C' -> ['A', 'B', 'C']
     # or if no AND, return expression as a list, e.g. 'A' -> ['A']
     if isinstance(expr, sqlglot.exp.And):
-        return list(expr.flatten())
+        return [cast(Expression, subclause) for subclause in expr.flatten()]
     return [expr]
 
 
@@ -660,11 +660,10 @@ class ComparisonLevel:
 
     def _match_weight_sql(self, gamma_column_name: str) -> str:
         sql = f"""
-        WHEN
-        {gamma_column_name} = {self.comparison_vector_value}
-        THEN cast({self._match_weight} as float8)
+        WHEN {gamma_column_name} = {self.comparison_vector_value} THEN
+            cast({self._match_weight} as float8)
         """
-        return dedent(sql)
+        return dedent(sql).strip()
 
     def _tf_adjustment_sql(
         self, gamma_column_name: str, comparison_levels: list[ComparisonLevel]
@@ -689,7 +688,9 @@ class ComparisonLevel:
             or self._tf_adjustment_weight == 0
             or self._is_else_level
         ):
-            sql = f"WHEN  {gamma_colname_value_is_this_level} then cast(0 as float8)"
+            sql = f"""
+            WHEN {gamma_colname_value_is_this_level} THEN cast(0 as float8)
+            """
         else:
             tf_adj_col = self._tf_adjustment_input_column
 
@@ -714,14 +715,16 @@ class ComparisonLevel:
 
             log2_u_prob = math.log2(u_prob_exact_match)
 
-            sql = f"""WHEN  {gamma_colname_value_is_this_level} then
-            (CASE WHEN {tf_adjustment_exists}
-            THEN
-            cast({self._tf_adjustment_weight} as float8) * (
-                cast({log2_u_prob} as float8) - log2({tf_u_value_sql})
-            )
-            ELSE cast(0 as float8)
-            END)"""
+            sql = f"""
+            WHEN {gamma_colname_value_is_this_level} THEN
+                CASE
+                    WHEN {tf_adjustment_exists} THEN
+                        cast({self._tf_adjustment_weight} as float8) * (
+                            cast({log2_u_prob} as float8) - log2({tf_u_value_sql})
+                        )
+                    ELSE cast(0 as float8)
+                END
+            """
         return dedent(sql).strip()
 
     def as_dict(self):

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/comparison_vector_values.py

@@ -4,6 +4,7 @@ import logging
 from typing import List, Optional
 
 from splink.internals.input_column import InputColumn
+from splink.internals.misc import indent_sql
 from splink.internals.unique_id_concat import (
     _composite_unique_id_from_nodes_sql,
 )
@@ -22,15 +23,15 @@ def compute_comparison_vector_values_sql(
     See [the fastlink paper](https://imai.fas.harvard.edu/research/files/linkage.pdf)
     for more details of what is meant by comparison vectors.
     """
-    select_cols_expr = ",".join(columns_to_select_for_comparison_vector_values)
-
+    select_columns = list(columns_to_select_for_comparison_vector_values)
     if include_clerical_match_score:
-        clerical_match_score = ", clerical_match_score"
-    else:
-        clerical_match_score = ""
+        select_columns.append("clerical_match_score")
+
+    select_cols_expr = ",\n".join(indent_sql(col) for col in select_columns)
 
     sql = f"""
-    select {select_cols_expr} {clerical_match_score}
+    select
+{select_cols_expr}
     from __splink__df_blocked
     """
 
@@ -61,7 +62,8 @@ def compute_comparison_vector_values_from_id_pairs_sqls(
     else:
         unique_id_columns = [unique_id_input_column]
 
-    select_cols_expr = ", \n".join(columns_to_select_for_blocking)
+    select_columns = [*columns_to_select_for_blocking, "b.match_key"]
+    select_cols_expr = ",\n".join(indent_sql(col) for col in select_columns)
 
     # Where there are large numbers of unmatched records, the DuckDB query planner
     # can struggle with the double inner join below.  It should
@@ -101,7 +103,8 @@ def compute_comparison_vector_values_from_id_pairs_sqls(
     # using the __splink__blocked_id_pairs as an associated (junction) table
     # That is, it does the join, but doesn't compute the comparison vectors
     sql = f"""
-    select {select_cols_expr}, b.match_key
+    select
+{select_cols_expr}
     from __splink__blocked_id_pairs as b
     inner join {input_tablename_l} as l
     on {uid_l_expr} = b.join_key_l
@@ -111,16 +114,16 @@ def compute_comparison_vector_values_from_id_pairs_sqls(
 
     sqls.append({"sql": sql, "output_table_name": "blocked_with_cols"})
 
-    select_cols_expr = ", \n".join(columns_to_select_for_comparison_vector_values)
-
+    select_columns = list(columns_to_select_for_comparison_vector_values)
     if include_clerical_match_score:
-        clerical_match_score = ", clerical_match_score"
-    else:
-        clerical_match_score = ""
+        select_columns.append("clerical_match_score")
+
+    select_cols_expr = ",\n".join(indent_sql(col) for col in select_columns)
 
     # The second table computes the comparison vectors from these aliases
     sql = f"""
-    select {select_cols_expr} {clerical_match_score}
+    select
+{select_cols_expr}
     from blocked_with_cols
     """
 

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/completeness.py

@@ -7,6 +7,7 @@ from splink.internals.charts import (
 )
 from splink.internals.database_api import DatabaseAPISubClass
 from splink.internals.input_column import InputColumn
+from splink.internals.misc import join_sql_with_union_all
 from splink.internals.pipeline import CTEPipeline
 from splink.internals.splink_dataframe import SplinkDataFrame
 from splink.internals.splinkdataframe_utils import (
@@ -75,7 +76,7 @@ def completeness_data(
         """
         sqls.append(sql)
 
-    sql = " union all ".join(sqls)
+    sql = join_sql_with_union_all(sqls)
 
     pipeline.enqueue_sql(sql, "__splink__df_all_column_completeness")
 

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/connected_components.py

@@ -16,6 +16,7 @@ import time
 from typing import Optional
 
 from splink.internals.database_api import DatabaseAPISubClass
+from splink.internals.misc import join_sql_with_union_all
 from splink.internals.pipeline import CTEPipeline
 from splink.internals.splink_dataframe import SplinkDataFrame
 
@@ -316,7 +317,7 @@ def solve_connected_components(
 
     pipeline = CTEPipeline()
 
-    sql = " UNION ALL ".join(
+    sql = join_sql_with_union_all(
         [
             f"""select node_id as {node_id_column_name}, representative as cluster_id
             from {t.physical_name}"""

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/database_api.py

@@ -16,7 +16,6 @@ from typing import (
 )
 
 import duckdb
-import sqlglot
 
 from splink.internals.cache_dict_with_logging import CacheDictWithLogging
 from splink.internals.logging_messages import execute_sql_logging_message_info, log_sql
@@ -106,16 +105,6 @@ class DatabaseAPI(ABC, Generic[TablishType]):
         try:
             return self._execute_sql_against_backend(final_sql)
         except Exception as e:
-            # Parse our SQL through sqlglot to pretty print
-            try:
-                final_sql = sqlglot.parse_one(
-                    final_sql,
-                    read=self.sql_dialect.sqlglot_dialect,
-                ).sql(pretty=True)
-                # if sqlglot produces any errors, just report the raw SQL
-            except Exception:
-                pass
-
             raise SplinkException(
                 f"Error executing the following sql for table "
                 f"`{templated_name}`({physical_name}):\n{final_sql}"
@@ -311,7 +300,7 @@ class DatabaseAPI(ABC, Generic[TablishType]):
         # returns sql
         # sensible default:
         self.delete_table_from_database(physical_name)
-        sql = f"CREATE TABLE {physical_name} AS {sql}"
+        sql = f"CREATE TABLE {physical_name} AS\n{sql}"
         return sql
 
     def _cleanup_for_execute_sql(

splink-5.0.0.dev3/splink/internals/duckdb/database_api_with_profiling.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import re
+from os import PathLike
+from pathlib import Path
+from typing import Union
+
+import duckdb
+
+from .database_api import DuckDBAPI
+from .dataframe import DuckDBDataFrame
+
+
+class DuckDBAPIWithProfiling(DuckDBAPI):
+    def __init__(
+        self,
+        connection: Union[str, duckdb.DuckDBPyConnection] = ":memory:",
+        output_schema: str = None,
+        query_profiling_dir: str | PathLike[str] = "tmp_query_profiling",
+    ):
+        super().__init__(connection=connection, output_schema=output_schema)
+        self.query_profiling_dir = Path(query_profiling_dir)
+        self.query_profiling_dir.mkdir(parents=True, exist_ok=True)
+        self._query_profile_counter = 0
+        self._pending_profile_path: Path | None = None
+        self._pending_profile_sql: str | None = None
+
+    def _should_profile_sql(self, sql: str) -> bool:
+        stripped_sql = sql.lstrip().upper()
+        return stripped_sql.startswith("SELECT") or stripped_sql.startswith("WITH")
+
+    def _next_query_profile_path(self, templated_name: str) -> Path:
+        self._query_profile_counter += 1
+        safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", templated_name).strip("_")
+        if not safe_name:
+            safe_name = "query"
+        filename = f"{self._query_profile_counter:04d}_{safe_name}_duckdb.txt"
+        return self.query_profiling_dir / filename
+
+    def _setup_for_execute_sql(self, sql: str, physical_name: str) -> str:
+        if self._should_profile_sql(sql):
+            self._pending_profile_path = self._next_query_profile_path(physical_name)
+            self._pending_profile_sql = sql
+        else:
+            self._pending_profile_path = None
+            self._pending_profile_sql = None
+
+        return super()._setup_for_execute_sql(sql, physical_name)
+
+    def _cleanup_for_execute_sql(
+        self, table: duckdb.DuckDBPyRelation, templated_name: str, physical_name: str
+    ) -> DuckDBDataFrame:
+        try:
+            output_df = self.table_to_splink_dataframe(templated_name, physical_name)
+            if self._pending_profile_path is not None and self._pending_profile_sql:
+                explain_result = super()._execute_sql_against_backend(
+                    f"EXPLAIN ANALYZE {self._pending_profile_sql}"
+                )
+                rows = explain_result.fetchall()
+                if len(rows) == 1 and len(rows[0]) == 2:
+                    profile_text = rows[0][1]
+                else:
+                    profile_text = "\n".join(
+                        " | ".join("" if value is None else str(value) for value in row)
+                        for row in rows
+                    )
+                self._pending_profile_path.write_text(profile_text, encoding="utf-8")
+            return output_df
+        finally:
+            self._pending_profile_path = None
+            self._pending_profile_sql = None

{splink-5.0.0.dev2 → splink-5.0.0.dev3}/splink/internals/em_training_session.py

@@ -26,7 +26,10 @@ from splink.internals.settings import (
     Settings,
     TrainingSettings,
 )
-from splink.internals.vertically_concatenate import compute_df_concat_with_tf
+from splink.internals.vertically_concatenate import (
+    enqueue_df_concat,
+    enqueue_df_concat_with_tf,
+)
 
 from .database_api import DatabaseAPISubClass
 from .exceptions import EMTrainingException
@@ -195,13 +198,12 @@ class EMTrainingSession:
         self._training_log_message()
 
         pipeline = CTEPipeline()
-        nodes_with_tf = compute_df_concat_with_tf(self._original_linker, pipeline)
-        pipeline = CTEPipeline([nodes_with_tf])
+        enqueue_df_concat(self._original_linker, pipeline)
 
         orig_settings = self._original_linker._settings_obj
         sqls = block_using_rules_sqls(
-            input_tablename_l="__splink__df_concat_with_tf",
-            input_tablename_r="__splink__df_concat_with_tf",
+            input_tablename_l="__splink__df_concat",
+            input_tablename_r="__splink__df_concat",
             blocking_rules=[self._blocking_rule_for_training],
             link_type=orig_settings._link_type,
             source_dataset_input_column=orig_settings.column_info_settings.source_dataset_input_column,
@@ -211,7 +213,8 @@ class EMTrainingSession:
 
         blocked_pairs = self.db_api.sql_pipeline_to_splink_dataframe(pipeline)
 
-        pipeline = CTEPipeline([blocked_pairs, nodes_with_tf])
+        pipeline = CTEPipeline([blocked_pairs])
+        enqueue_df_concat_with_tf(self._original_linker, pipeline)
 
         sqls = compute_comparison_vector_values_from_id_pairs_sqls(
             orig_settings._columns_to_select_for_blocking,