metricflow 0.204.0.dev5__tar.gz → 0.204.0.dev7__tar.gz

metricflow-0.204.0.dev7/PKG-INFO

@@ -0,0 +1,170 @@
+Metadata-Version: 2.1
+Name: metricflow
+Version: 0.204.0.dev7
+Summary: Translates a simple metric definition into reusable SQL and executes it against the SQL engine of your choice.
+Project-URL: Documentation, https://transform.co/metricflow
+Project-URL: Source Code, https://github.com/transform-data/metricflow
+Author: dbt Labs
+License-Expression: BUSL-1.1
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: <3.12,>=3.8
+Requires-Dist: click>=7.1.2
+Requires-Dist: dbt-core~=1.7.0
+Requires-Dist: dbt-semantic-interfaces~=0.4.2
+Requires-Dist: graphviz<0.21,>=0.18.2
+Requires-Dist: halo~=0.0.31
+Requires-Dist: jinja2>=2.11.3
+Requires-Dist: markupsafe==2.0.1
+Requires-Dist: more-itertools<10.2.0,>=8.10.0
+Requires-Dist: numpy>=1.22.2
+Requires-Dist: pandas~=1.5.0
+Requires-Dist: pydantic~=1.10.0
+Requires-Dist: python-dateutil~=2.8.2
+Requires-Dist: pyyaml~=6.0
+Requires-Dist: rapidfuzz~=3.0
+Requires-Dist: ruamel-yaml~=0.17.21
+Requires-Dist: tabulate>=0.8.9
+Requires-Dist: typing-extensions~=4.4
+Requires-Dist: update-checker~=0.18.0
+Provides-Extra: dbt-bigquery
+Requires-Dist: dbt-bigquery~=1.7.0; extra == 'dbt-bigquery'
+Provides-Extra: dbt-databricks
+Requires-Dist: dbt-databricks~=1.7.0; extra == 'dbt-databricks'
+Provides-Extra: dbt-duckdb
+Requires-Dist: dbt-duckdb~=1.7.0; extra == 'dbt-duckdb'
+Provides-Extra: dbt-postgres
+Requires-Dist: dbt-postgres~=1.7.0; extra == 'dbt-postgres'
+Provides-Extra: dbt-redshift
+Requires-Dist: dbt-redshift~=1.7.0; extra == 'dbt-redshift'
+Provides-Extra: dbt-snowflake
+Requires-Dist: dbt-snowflake~=1.7.0; extra == 'dbt-snowflake'
+Provides-Extra: dbt-trino
+Requires-Dist: dbt-trino~=1.7.0; extra == 'dbt-trino'
+Provides-Extra: dev-packages
+Requires-Dist: click>=8.1.6; extra == 'dev-packages'
+Requires-Dist: mypy~=1.7.0; extra == 'dev-packages'
+Requires-Dist: pre-commit~=3.2.2; extra == 'dev-packages'
+Requires-Dist: pytest-mock~=3.7.0; extra == 'dev-packages'
+Requires-Dist: pytest-xdist~=3.2.1; extra == 'dev-packages'
+Requires-Dist: pytest~=7.1.1; extra == 'dev-packages'
+Requires-Dist: types-python-dateutil; extra == 'dev-packages'
+Requires-Dist: types-pyyaml; extra == 'dev-packages'
+Requires-Dist: types-tabulate; extra == 'dev-packages'
+Provides-Extra: sql-client-packages
+Requires-Dist: sqlalchemy2-stubs~=0.0.2a21; extra == 'sql-client-packages'
+Requires-Dist: sqlalchemy~=1.4.42; extra == 'sql-client-packages'
+Provides-Extra: trino-sql-client-packages
+Requires-Dist: trino~=0.327.0; extra == 'trino-sql-client-packages'
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <a target="_blank" href="https://transform.co/metricflow">
+    <picture>
+      <img  alt="metricflow logo" src="https://github.com/dbt-labs/metricflow/raw/main/assets/MetricFlow_logo.png" width="auto" height="120">
+    </picture>
+  </a>
+  <br /><br />
+  <b>Build and maintain all of your metric logic in code.</b>
+  <br /><br />
+  <a target="_blank" href="https://twitter.com/dbt_labs">
+    <img src="https://img.shields.io/twitter/follow/dbt_labs?labelColor=image.png&color=163B36&logo=twitter&style=flat">
+  </a>
+  <a target="_blank" href="https://www.getdbt.com/community/">
+    <img src="https://img.shields.io/badge/Slack-join-163B36">
+  </a>
+  <a target="_blank" href="https://github.com/dbt-labs/metricflow">
+    <img src="https://img.shields.io/github/stars/dbt-labs/metricflow?labelColor=image.png&color=163B36&logo=github">
+  </a>
+  <br />
+  <a target="_blank" href="https://github.com/dbt-labs/metricflow/blob/master/LICENSE">
+    <img src="https://img.shields.io/pypi/l/metricflow?color=163B36&logo=AGPL-3.0">
+  </a>
+  <a target="_blank" href="https://pypi.org/project/metricflow/">
+    <img src="https://img.shields.io/pypi/v/metricflow?labelColor=&color=163B36">
+  </a>
+  <img src="https://img.shields.io/pypi/pyversions/metricflow?labelColor=&color=163B36">
+</p>
+
+# Welcome to MetricFlow
+
+See our latest updates in the [Metricflow Changelog](https://github.com/dbt-labs/metricflow/blob/main/CHANGELOG.md)!
+
+MetricFlow is a semantic layer that makes it easy to organize metric definitions. It takes those definitions and generates legible and reusable SQL. This makes it easy to get consistent metrics output broken down by attributes (dimensions) of interest.
+
+The name comes from the approach taken to generate metrics. A query is compiled into a query plan (represented below) called a dataflow that constructs metrics. The plan is then optimized and rendered to engine-specific SQL.
+
+<p align="center">
+<img src="https://github.com/dbt-labs/metricflow/raw/main/assets/example_plan.svg" height="500"/>
+<br /><br />
+</p>
+
+MetricFlow provides a set of abstractions that help you construct complicated logic and dynamically generate queries to handle:
+
+- Multi-hop joins between fact and dimension sources
+- Complex metric types such as ratio, expression, and cumulative
+- Metric aggregation to different time granularities
+- And so much more
+
+To get up and running with your own metrics, you should rely on [MetricFlow’s documentation](https://docs.getdbt.com/docs/build/build-metrics-intro).
+
+## Licensing
+
+MetricFlow is distributed under a Business Source License (BUSL-1.1). For details on our additional use grant, change license, and change date please refer to our [licensing agreement](https://github.com/dbt-labs/metricflow/blob/main/LICENSE).
+
+## Getting Started
+
+### Install MetricFlow
+
+MetricFlow can be installed from PyPi for use as a Python library with the following command:
+
+```
+pip install dbt-metricflow
+```
+
+MetricFlow currently serves as a query compilation and SQL rendering library, built to work in conjunction with a dbt project. As such, using MetricFlow requires a working dbt project and a dbt adapter. We provide the `dbt-metricflow` bundle for this purpose. You may choose to install other adapters as optional extras from dbt-metricflow.
+
+You may need to install Postgres or Graphviz. You can do so by following the install instructions for [Postgres](https://www.postgresql.org/download/) or [Graphviz](https://www.graphviz.org/download/). Mac users may prefer to use brew: `brew install postgresql` or `brew install graphviz`.
+
+### Tutorial
+
+The best way to get started is to follow the tutorial steps, which you can access by running:
+
+```
+mf tutorial
+```
+
+Note: this must be run from a dbt project root directory.
+
+## Resources
+
+- [Website](https://transform.co/metricflow)
+- [Documentation](https://docs.getdbt.com/docs/build/build-metrics-intro)
+- [Slack Community](https://www.getdbt.com/community/)
+- [MetricFlow Git Repository](https://github.com/dbt-labs/metricflow)
+- [CHANGELOG.md](https://github.com/dbt-labs/metricflow/blob/main/CHANGELOG.md)
+- [TENETS.md](https://github.com/dbt-labs/metricflow/blob/main/TENETS.md)
+
+## Contributing and Code of Conduct
+
+This project will be a place where people can easily contribute high-quality updates in a supportive environment.
+
+Please read our [code of conduct](https://docs.getdbt.com/community/resources/code-of-conduct) before diving in.
+
+To get started on direct contributions, head on over to our [contributor guide](https://github.com/dbt-labs/metricflow/blob/main/CONTRIBUTING.md).
+
+## License
+
+MetricFlow is source-available software.
+
+Version 0 to 0.140.0 was covered by the Affero GPL license.
+Version 0.150.0 and greater is covered by the BSL license..
+
+MetricFlow is built by [dbt Labs](https://www.getdbt.com/).

metricflow-0.204.0.dev7/metricflow/cli/dbt_connectors/adapter_backed_client.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import enum
+import logging
+import time
+
+import pandas as pd
+from dbt.adapters.base.impl import BaseAdapter
+from dbt.exceptions import DbtDatabaseError
+from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
+from dbt_semantic_interfaces.pretty_print import pformat_big_objects
+
+from metricflow.errors.errors import SqlBindParametersNotSupportedError
+from metricflow.mf_logging.formatting import indent
+from metricflow.protocols.sql_client import SqlEngine
+from metricflow.random_id import random_id
+from metricflow.sql.render.big_query import BigQuerySqlQueryPlanRenderer
+from metricflow.sql.render.databricks import DatabricksSqlQueryPlanRenderer
+from metricflow.sql.render.duckdb_renderer import DuckDbSqlQueryPlanRenderer
+from metricflow.sql.render.postgres import PostgresSQLSqlQueryPlanRenderer
+from metricflow.sql.render.redshift import RedshiftSqlQueryPlanRenderer
+from metricflow.sql.render.snowflake import SnowflakeSqlQueryPlanRenderer
+from metricflow.sql.render.sql_plan_renderer import SqlQueryPlanRenderer
+from metricflow.sql.render.trino import TrinoSqlQueryPlanRenderer
+from metricflow.sql.sql_bind_parameters import SqlBindParameters
+from metricflow.sql_request.sql_request_attributes import SqlJsonTag, SqlRequestId, SqlRequestTagSet
+from metricflow.sql_request.sql_statement_metadata import CombinedSqlTags, SqlStatementCommentMetadata
+
+logger = logging.getLogger(__name__)
+
+
+# Discovered via trial and error from the original, and now defunct DatabricksSqlClient implementation
+DATABRICKS_SQL_WAREHOUSE_EXPLAIN_PLAN_ERROR_KEY = "Error occurred during query planning"
+DATABRICKS_CLUSTER_EXPLAIN_PLAN_ERROR_KEY = "org.apache.spark.sql.AnalysisException"
+
+
+class SupportedAdapterTypes(enum.Enum):
+    """Enumeration of supported dbt adapter types."""
+
+    DATABRICKS = "databricks"
+    POSTGRES = "postgres"
+    SNOWFLAKE = "snowflake"
+    REDSHIFT = "redshift"
+    BIGQUERY = "bigquery"
+    DUCKDB = "duckdb"
+    TRINO = "trino"
+
+    @property
+    def sql_engine_type(self) -> SqlEngine:
+        """Return the SqlEngine corresponding to the supported adapter type."""
+        if self is SupportedAdapterTypes.BIGQUERY:
+            return SqlEngine.BIGQUERY
+        elif self is SupportedAdapterTypes.DATABRICKS:
+            return SqlEngine.DATABRICKS
+        elif self is SupportedAdapterTypes.POSTGRES:
+            return SqlEngine.POSTGRES
+        elif self is SupportedAdapterTypes.REDSHIFT:
+            return SqlEngine.REDSHIFT
+        elif self is SupportedAdapterTypes.SNOWFLAKE:
+            return SqlEngine.SNOWFLAKE
+        elif self is SupportedAdapterTypes.DUCKDB:
+            return SqlEngine.DUCKDB
+        elif self is SupportedAdapterTypes.TRINO:
+            return SqlEngine.TRINO
+        else:
+            assert_values_exhausted(self)
+
+    @property
+    def sql_query_plan_renderer(self) -> SqlQueryPlanRenderer:
+        """Return the SqlQueryPlanRenderer corresponding to the supported adapter type."""
+        if self is SupportedAdapterTypes.BIGQUERY:
+            return BigQuerySqlQueryPlanRenderer()
+        elif self is SupportedAdapterTypes.DATABRICKS:
+            return DatabricksSqlQueryPlanRenderer()
+        elif self is SupportedAdapterTypes.POSTGRES:
+            return PostgresSQLSqlQueryPlanRenderer()
+        elif self is SupportedAdapterTypes.REDSHIFT:
+            return RedshiftSqlQueryPlanRenderer()
+        elif self is SupportedAdapterTypes.SNOWFLAKE:
+            return SnowflakeSqlQueryPlanRenderer()
+        elif self is SupportedAdapterTypes.DUCKDB:
+            return DuckDbSqlQueryPlanRenderer()
+        elif self is SupportedAdapterTypes.TRINO:
+            return TrinoSqlQueryPlanRenderer()
+        else:
+            assert_values_exhausted(self)
+
+
+class AdapterBackedSqlClient:
+    """SqlClient implementation which delegates database operations to a dbt BaseAdapter instance.
+
+    This is a generic wrpaper class meant to cover all of our logging, querying, and internal configuration
+    needs while delegating all connection state management and warehouse communication work to an underlying
+    dbt adapter instance. This relies on BaseAdapter, rather than SQLAdapter, because BigQuery is an instance
+    of the more generic BaseAdapter class.
+    """
+
+    def __init__(self, adapter: BaseAdapter):
+        """Initializer sourced from a BaseAdapter instance.
+
+        The dbt BaseAdapter should already be fully initialized, including all credential verification, and
+        ready for use for establishing connections and issuing queries.
+        """
+        self._adapter = adapter
+        try:
+            adapter_type = SupportedAdapterTypes(self._adapter.type())
+        except ValueError as e:
+            raise ValueError(
+                f"Adapter type {self._adapter.type()} is not supported. Must be one "
+                f"of {[item.value for item in SupportedAdapterTypes]}."
+            ) from e
+
+        self._sql_engine_type = adapter_type.sql_engine_type
+        self._sql_query_plan_renderer = adapter_type.sql_query_plan_renderer
+        logger.info(f"Initialized AdapterBackedSqlClient with dbt adapter type `{adapter_type.value}`")
+
+    @property
+    def sql_engine_type(self) -> SqlEngine:
+        """An enumerated value representing the underlying SqlEngine supported by the dbt adapter for this instance."""
+        return self._sql_engine_type
+
+    @property
+    def sql_query_plan_renderer(self) -> SqlQueryPlanRenderer:
+        """Dialect-specific SQL query plan renderer used for converting MetricFlow's query plan to executable SQL."""
+        return self._sql_query_plan_renderer
+
+    def query(
+        self,
+        stmt: str,
+        sql_bind_parameters: SqlBindParameters = SqlBindParameters(),
+        extra_tags: SqlJsonTag = SqlJsonTag(),
+    ) -> pd.DataFrame:
+        """Query statement; result expected to be data which will be returned as a DataFrame.
+
+        Args:
+            stmt: The SQL query statement to run. This should produce output via a SELECT
+            sql_bind_parameters: The parameter replacement mapping for filling in
+                concrete values for SQL query parameters.
+            extra_tags: An object containing JSON serialized tags meant for annotating queries.
+        """
+        start = time.time()
+        request_id = SqlRequestId(f"mf_rid__{random_id()}")
+        combined_tags = AdapterBackedSqlClient._consolidate_tags(json_tags=extra_tags, request_id=request_id)
+        statement = SqlStatementCommentMetadata.add_tag_metadata_as_comment(
+            sql_statement=stmt, combined_tags=combined_tags
+        )
+        if sql_bind_parameters.param_dict:
+            raise SqlBindParametersNotSupportedError(
+                f"Invalid execute statement - we do not support queries with bind parameters through dbt adapters! "
+                f"Bind params: {sql_bind_parameters.param_dict}"
+            )
+        logger.info(AdapterBackedSqlClient._format_run_query_log_message(statement, sql_bind_parameters))
+        with self._adapter.connection_named(f"MetricFlow_request_{request_id}"):
+            # returns a Tuple[AdapterResponse, agate.Table] but the decorator converts it to Any
+            result = self._adapter.execute(sql=statement, auto_begin=True, fetch=True)
+            logger.info(f"Query returned from dbt Adapter with response {result[0]}")
+
+        agate_data = result[1]
+        df = pd.DataFrame([row.values() for row in agate_data.rows], columns=agate_data.column_names)
+        stop = time.time()
+        logger.info(f"Finished running the query in {stop - start:.2f}s with {df.shape[0]} row(s) returned")
+        return df
+
+    def execute(
+        self,
+        stmt: str,
+        sql_bind_parameters: SqlBindParameters = SqlBindParameters(),
+        extra_tags: SqlJsonTag = SqlJsonTag(),
+    ) -> None:
+        """Execute a SQL statement. No result will be returned.
+
+        Args:
+            stmt: The SQL query statement to run. This should not produce output.
+            sql_bind_parameters: The parameter replacement mapping for filling in
+                concrete values for SQL query parameters.
+            extra_tags: An object containing JSON serialized tags meant for annotating queries.
+        """
+        if sql_bind_parameters.param_dict:
+            raise SqlBindParametersNotSupportedError(
+                f"Invalid execute statement - we do not support execute commands with bind parameters through dbt "
+                f"adapters! Bind params: {SqlBindParameters.param_dict}"
+            )
+        start = time.time()
+        request_id = SqlRequestId(f"mf_rid__{random_id()}")
+        combined_tags = AdapterBackedSqlClient._consolidate_tags(json_tags=extra_tags, request_id=request_id)
+        statement = SqlStatementCommentMetadata.add_tag_metadata_as_comment(
+            sql_statement=stmt, combined_tags=combined_tags
+        )
+        logger.info(AdapterBackedSqlClient._format_run_query_log_message(statement, sql_bind_parameters))
+        with self._adapter.connection_named(f"MetricFlow_request_{request_id}"):
+            result = self._adapter.execute(statement, auto_begin=True, fetch=False)
+            # Calls to execute often involve some amount of DDL so we commit here
+            self._adapter.commit_if_has_connection()
+            logger.info(f"Query executed via dbt Adapter with response {result[0]}")
+        stop = time.time()
+        logger.info(f"Finished running the query in {stop - start:.2f}s")
+        return None
+
+    def dry_run(
+        self,
+        stmt: str,
+        sql_bind_parameters: SqlBindParameters = SqlBindParameters(),
+    ) -> None:
+        """Dry run statement; checks that the 'stmt' is queryable. Returns None on success.
+
+        Raises an exception if the 'stmt' isn't queryable.
+
+        Args:
+            stmt: The SQL query statement to dry run.
+            sql_bind_parameters: The parameter replacement mapping for filling in
+                concrete values for SQL query parameters.
+        """
+        start = time.time()
+        logger.info(
+            f"Running dry_run of:"
+            f"\n\n{indent(stmt)}\n"
+            + (f"\nwith parameters: {dict(sql_bind_parameters.param_dict)}" if sql_bind_parameters.param_dict else "")
+        )
+        request_id = SqlRequestId(f"mf_rid__{random_id()}")
+        connection_name = f"MetricFlow_dry_run_request_{request_id}"
+        # TODO - consolidate to self._adapter.validate_sql() when all implementations will work from within MetricFlow
+
+        # Trino has a bug where explain command actually creates table. Wrapping with validate to avoid this.
+        # See https://github.com/trinodb/trino/issues/130
+        if self.sql_engine_type is SqlEngine.TRINO:
+            with self._adapter.connection_named(connection_name):
+                # Either the response will be bool value or a string with error message from Trino.
+                result = self._adapter.execute(f"EXPLAIN (type validate) {stmt}", auto_begin=True, fetch=True)
+                has_error = False if str(result[0]) == "SUCCESS" else True
+                if has_error:
+                    raise DbtDatabaseError("Encountered error in Trino dry run.")
+
+        elif self.sql_engine_type is SqlEngine.BIGQUERY:
+            with self._adapter.connection_named(connection_name):
+                self._adapter.validate_sql(stmt)
+        else:
+            is_databricks = self.sql_engine_type is SqlEngine.DATABRICKS
+            with self._adapter.connection_named(connection_name):
+                results = self._adapter.execute(f"EXPLAIN {stmt}", auto_begin=True, fetch=is_databricks)
+
+            if is_databricks:
+                # We have to parse the output results from adapter.execute, but only in Databricks. We should probably use
+                # a subclass for this and override things properly, but there's hopefully an upstream fix in our near
+                # future so let's just put this ugly bit in for now.
+                plan_output_str = "\n".join([",".join(row.values()) for row in results[1].rows])
+                has_error = (
+                    plan_output_str.find(DATABRICKS_CLUSTER_EXPLAIN_PLAN_ERROR_KEY) != -1
+                    or plan_output_str.find(DATABRICKS_SQL_WAREHOUSE_EXPLAIN_PLAN_ERROR_KEY) != -1
+                )
+                if has_error:
+                    raise DbtDatabaseError(f"Encountered error in Databricks dry run. Full output: {plan_output_str}")
+
+        stop = time.time()
+        logger.info(f"Finished running the dry_run in {stop - start:.2f}s")
+        return
+
+    def close(self) -> None:  # noqa: D
+        self._adapter.cancel_open_connections()
+
+    def render_bind_parameter_key(self, bind_parameter_key: str) -> str:
+        """Wrap execution parameter key with syntax accepted by engine."""
+        raise SqlBindParametersNotSupportedError(
+            "We do not support queries with bind parameters through dbt adapters, so we do not have rendering enabled!"
+        )
+
+    @staticmethod
+    def _format_run_query_log_message(statement: str, sql_bind_parameters: SqlBindParameters) -> str:
+        """Helper for creating nicely formatted query logging."""
+        message = f"Running query:\n\n{indent(statement)}"
+        if len(sql_bind_parameters.param_dict) > 0:
+            message += (
+                f"\n" f"\n" f"with parameters:\n" f"\n" f"{indent(pformat_big_objects(sql_bind_parameters.param_dict))}"
+            )
+        return message
+
+    @staticmethod
+    def _consolidate_tags(json_tags: SqlJsonTag, request_id: SqlRequestId) -> CombinedSqlTags:
+        """Consolidates json tags and request ID into a single set of tags."""
+        return CombinedSqlTags(
+            system_tags=SqlRequestTagSet().add_request_id(request_id=request_id),
+            extra_tag=json_tags,
+        )

metricflow-0.204.0.dev7/metricflow/dag/id_generation.py

@@ -0,0 +1,101 @@
+"""Prefixes to use when generating IDs. Listed here to avoid conflicts."""
+
+# Nodes in a DAG have IDs of the form "prefix_number". These are the prefixes used for the different node types.
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Dict, Type
+
+DATAFLOW_NODE_AGGREGATE_MEASURES_ID_PREFIX = "am"
+DATAFLOW_NODE_COMPUTE_METRICS_ID_PREFIX = "cm"
+DATAFLOW_NODE_JOIN_AGGREGATED_MEASURES_BY_GROUPBY_COLUMNS_PREFIX = "jamgc"
+DATAFLOW_NODE_JOIN_TO_STANDARD_OUTPUT_ID_PREFIX = "jso"
+DATAFLOW_NODE_JOIN_SELF_OVER_TIME_RANGE_ID_PREFIX = "jotr"
+DATAFLOW_NODE_ORDER_BY_LIMIT_ID_PREFIX = "obl"
+DATAFLOW_NODE_PASS_FILTER_ELEMENTS_ID_PREFIX = "pfe"
+DATAFLOW_NODE_READ_SQL_SOURCE_ID_PREFIX = "rss"
+DATAFLOW_NODE_WHERE_CONSTRAINT_ID_PREFIX = "wcc"
+DATAFLOW_NODE_WRITE_TO_RESULT_DATAFRAME_ID_PREFIX = "wrd"
+DATAFLOW_NODE_WRITE_TO_RESULT_TABLE_ID_PREFIX = "wrt"
+DATAFLOW_NODE_COMBINE_AGGREGATED_OUTPUTS_ID_PREFIX = "cao"
+DATAFLOW_NODE_CONSTRAIN_TIME_RANGE_ID_PREFIX = "ctr"
+DATAFLOW_NODE_SET_MEASURE_AGGREGATION_TIME = "sma"
+DATAFLOW_NODE_SEMI_ADDITIVE_JOIN_ID_PREFIX = "saj"
+DATAFLOW_NODE_JOIN_TO_TIME_SPINE_ID_PREFIX = "jts"
+DATAFLOW_NODE_MIN_MAX_ID_PREFIX = "mm"
+DATAFLOW_NODE_ADD_UUID_COLUMN_PREFIX = "auid"
+DATAFLOW_NODE_JOIN_CONVERSION_EVENTS_PREFIX = "jce"
+
+SQL_EXPR_COLUMN_REFERENCE_ID_PREFIX = "cr"
+SQL_EXPR_COMPARISON_ID_PREFIX = "cmp"
+SQL_EXPR_FUNCTION_ID_PREFIX = "fnc"
+SQL_EXPR_PERCENTILE_ID_PREFIX = "perc"
+SQL_EXPR_STRING_ID_PREFIX = "str"
+SQL_EXPR_NULL_PREFIX = "null"
+SQL_EXPR_LOGICAL_OPERATOR_PREFIX = "lo"
+SQL_EXPR_STRING_LITERAL_PREFIX = "sl"
+SQL_EXPR_IS_NULL_PREFIX = "isn"
+SQL_EXPR_CAST_TO_TIMESTAMP_PREFIX = "ctt"
+SQL_EXPR_DATE_TRUNC = "dt"
+SQL_EXPR_SUBTRACT_TIME_INTERVAL_PREFIX = "sti"
+SQL_EXPR_EXTRACT = "ex"
+SQL_EXPR_RATIO_COMPUTATION = "rc"
+SQL_EXPR_BETWEEN_PREFIX = "betw"
+SQL_EXPR_WINDOW_FUNCTION_ID_PREFIX = "wfnc"
+SQL_EXPR_GENERATE_UUID_PREFIX = "uuid"
+
+SQL_PLAN_SELECT_STATEMENT_ID_PREFIX = "ss"
+SQL_PLAN_TABLE_FROM_CLAUSE_ID_PREFIX = "tfc"
+
+EXEC_NODE_READ_SQL_QUERY = "rsq"
+EXEC_NODE_NOOP = "noop"
+EXEC_NODE_WRITE_TO_TABLE = "wtt"
+
+DATAFLOW_PLAN_PREFIX = "dfp"
+OPTIMIZED_DATAFLOW_PLAN_PREFIX = "dfpo"
+SQL_QUERY_PLAN_PREFIX = "sqp"
+EXEC_PLAN_PREFIX = "ep"
+
+MF_DAG = "mfd"
+
+
+logger = logging.getLogger(__name__)
+
+
+class IdGenerator:
+    """Helps generate unique ID strings."""
+
+    def __init__(self, start_value: int = 0) -> None:  # noqa: D
+        """Identifiers of the form (prefix + number) e.g. my_node_1."""
+        self._next_id = start_value
+        self._lock = threading.Lock()
+
+    def create_id(self, prefix: str) -> str:
+        """Create a unique ID string that can be used for naming nodes (or others) using the form <prefix>_<number>."""
+        with self._lock:
+            new_id = f"{prefix}_{self._next_id}"
+            self._next_id += 1
+            return new_id
+
+
+class IdGeneratorRegistry:
+    """Enumerate all IdGenerators used so that they can be patched appropriately in testing.
+
+    See: patch_id_generators()
+    """
+
+    DEFAULT_START_VALUE = 0
+    _state_lock = threading.Lock()
+    _class_name_to_id_generator: Dict[str, IdGenerator] = {}
+
+    @classmethod
+    def for_class(cls, class_type: Type) -> IdGenerator:
+        """Return an ID generator for the given class."""
+        with cls._state_lock:
+            class_name = ".".join([class_type.__module__, class_type.__name__])
+            if class_name not in cls._class_name_to_id_generator:
+                cls._class_name_to_id_generator[class_name] = IdGenerator(
+                    start_value=IdGeneratorRegistry.DEFAULT_START_VALUE
+                )
+            return cls._class_name_to_id_generator[class_name]