datajunction-server 0.0.1a64.dev1__tar.gz → 0.0.1a64.dev3__tar.gz

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/.env

@@ -1,2 +1,3 @@
 QUERY_SERVICE=http://djqs:8001
 SECRET=a-fake-secretkey
+NODE_LIST_MAX=10000

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/.gitignore

@@ -123,3 +123,5 @@ Untitled*
 
 # postgres
 postgres_metadata
+node_modules
+node_modules

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a64.dev1
+Version: 0.0.1a64.dev3
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a64.dev3/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a64.dev3"

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/api/helpers.py

@@ -198,6 +198,7 @@ async def get_query(  # pylint: disable=too-many-arguments
     limit: Optional[int] = None,
     engine: Optional[Engine] = None,
     access_control: Optional[access.AccessControlStore] = None,
+    use_materialized: bool = True,
 ) -> ast.Query:
     """
     Get a query for a metric, dimensions, and filters
@@ -208,7 +209,11 @@ async def get_query(  # pylint: disable=too-many-arguments
 
     node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)
     build_criteria = get_default_criteria(node.current, engine)  # type: ignore
-    query_builder = await QueryBuilder.create(session, node.current)  # type: ignore
+    query_builder = await QueryBuilder.create(
+        session,
+        node.current,  # type: ignore
+        use_materialized=use_materialized,
+    )
     query_ast = await (
         query_builder.ignore_errors()
         .with_access_control(access_control)
@@ -370,6 +375,7 @@ async def validate_cube(  # pylint: disable=too-many-locals
                     selectinload(NodeRevision.parents),
                 ),
             ],
+            include_inactive=False,
         ),
         key=lambda x: metrics_sorting_order.get(x.name, 0),
     )
@@ -597,8 +603,8 @@ async def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,
     engine_name: Optional[str] = None,
     engine_version: Optional[str] = None,
     access_control: Optional[access.AccessControlStore] = None,
-    use_materialized: bool = True,
     ignore_errors: bool = True,
+    use_materialized: bool = True,
 ) -> Tuple[TranslatedSQL, Engine, Catalog]:
     """
     Build SQL for multiple metrics. Used by both /sql and /data endpoints

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/api/nodes.py

@@ -25,7 +25,6 @@ from datajunction_server.api.helpers import (
 )
 from datajunction_server.api.namespaces import create_node_namespace
 from datajunction_server.api.tags import get_tags_by_name
-from datajunction_server.constants import NODE_LIST_MAX
 from datajunction_server.database import DimensionLink
 from datajunction_server.database.attributetype import ColumnAttribute
 from datajunction_server.database.column import Column
@@ -280,16 +279,17 @@ async def list_all_nodes_with_details(
             Node.type == node_type if node_type else True,
             is_(Node.deactivated_at, None),
         )
-        .limit(NODE_LIST_MAX)
+        .order_by(NodeRevision.updated_at.desc())
+        .limit(settings.node_list_max)
     )  # Very high limit as a safeguard
     results = [
         NodeIndexItem(name=row[0], display_name=row[1], description=row[2], type=row[3])
         for row in (await session.execute(nodes_query)).all()
     ]
-    if len(results) == NODE_LIST_MAX:  # pragma: no cover
+    if len(results) == settings.node_list_max:  # pragma: no cover
         _logger.warning(
             "%s limit reached when returning all nodes, all nodes may not be captured in results",
-            NODE_LIST_MAX,
+            settings.node_list_max,
         )
     approvals = [
         approval.access_object.name

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/api/sql.py

@@ -64,6 +64,7 @@ async def get_measures_sql_for_cube_v2(
     validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
         validate_access,
     ),
+    use_materialized: bool = True,
 ) -> List[GeneratedSQL]:
     """
     Return measures SQL for a set of metrics with dimensions and filters.
@@ -92,6 +93,7 @@ async def get_measures_sql_for_cube_v2(
         validate_access=validate_access,
         include_all_columns=include_all_columns,
         sql_transpilation_library=settings.sql_transpilation_library,
+        use_materialized=use_materialized,
     )
     return measures_query
 
@@ -107,6 +109,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
     engine: Engine,
     access_control: AccessControlStore,
     ignore_errors: bool = True,
+    use_materialized: bool = True,
 ) -> QueryRequest:
     """
     Build node SQL and save it to query requests
@@ -132,6 +135,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
             engine_name=engine.name if engine else None,
             engine_version=engine.version if engine else None,
             access_control=access_control,
+            use_materialized=use_materialized,
         )
         # We save the request for both the cube and the metrics, so that if someone makes either
         # of these types of requests, they'll go to the cached query
@@ -169,6 +173,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
             engine.version if engine else None,
             access_control=access_control,
             ignore_errors=ignore_errors,
+            use_materialized=use_materialized,
         )
         query = translated_sql.sql
         columns = translated_sql.columns
@@ -182,6 +187,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
             limit=limit,
             engine=engine,
             access_control=access_control,
+            use_materialized=use_materialized,
         )
         columns = [
             assemble_column_metadata(col)  # type: ignore
@@ -219,6 +225,7 @@ async def get_node_sql(  # pylint: disable=too-many-locals
     validate_access: access.ValidateAccessFn,  # pylint: disable=redefined-outer-name
     background_tasks: BackgroundTasks,
     ignore_errors: bool = True,
+    use_materialized: bool = True,
 ) -> Tuple[TranslatedSQL, QueryRequest]:
     """
     Return SQL for a node.
@@ -259,6 +266,7 @@ async def get_node_sql(  # pylint: disable=too-many-locals
             session=session,
             engine=engine,
             access_control=access_control,
+            use_materialized=use_materialized,
         )
         return (
             TranslatedSQL(
@@ -279,6 +287,7 @@ async def get_node_sql(  # pylint: disable=too-many-locals
         engine=engine,  # type: ignore
         access_control=access_control,
         ignore_errors=ignore_errors,
+        use_materialized=use_materialized,
     )
     return (
         TranslatedSQL(
@@ -311,6 +320,7 @@ async def get_sql(  # pylint: disable=too-many-locals
     ),
     background_tasks: BackgroundTasks,
     ignore_errors: Optional[bool] = True,
+    use_materialized: Optional[bool] = True,
 ) -> TranslatedSQL:
     """
     Return SQL for a node.
@@ -328,12 +338,13 @@ async def get_sql(  # pylint: disable=too-many-locals
         validate_access=validate_access,
         background_tasks=background_tasks,
         ignore_errors=ignore_errors,  # type: ignore
+        use_materialized=use_materialized,  # type: ignore
     )
     return translated_sql
 
 
 @router.get("/sql/", response_model=TranslatedSQL, name="Get SQL For Metrics")
-async def get_sql_for_metrics(
+async def get_sql_for_metrics(  # pylint: disable=too-many-locals
     metrics: List[str] = Query([]),
     dimensions: List[str] = Query([]),
     filters: List[str] = Query([]),
@@ -348,6 +359,7 @@ async def get_sql_for_metrics(
         validate_access,
     ),
     ignore_errors: Optional[bool] = True,
+    use_materialized: Optional[bool] = True,
 ) -> TranslatedSQL:
     """
     Return SQL for a set of metrics with dimensions and filters
@@ -392,6 +404,7 @@ async def get_sql_for_metrics(
         engine_version,
         access_control,
         ignore_errors=ignore_errors,  # type: ignore
+        use_materialized=use_materialized,  # type: ignore
     )
 
     await QueryRequest.save_query_request(

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/config.py

@@ -91,6 +91,8 @@ class Settings(
     # Interval in seconds with which to expire caching of any indexes
     index_cache_expire = 60
 
+    default_catalog_id: int = 0
+
     # SQLAlchemy engine config
     db_pool_size = 20
     db_max_overflow = 20
@@ -103,6 +105,9 @@ class Settings(
     db_keepalives_interval = 10
     db_keepalives_count = 5
 
+    # Maximum amount of nodes to return for requests to list all nodes
+    node_list_max = 10000
+
     @property
     def celery(self) -> Celery:
         """

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/constants.py

@@ -18,6 +18,3 @@ GET_COLUMNS_TIMEOUT = timedelta(seconds=60)
 
 AUTH_COOKIE = "__dj"
 LOGGED_IN_FLAG_COOKIE = "__djlif"
-
-# Maximum amount of nodes to return for requests to list all nodes
-NODE_LIST_MAX = 1000

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/construction/build_v2.py

@@ -101,6 +101,7 @@ async def get_measures_query(  # pylint: disable=too-many-locals
     cast_timestamp_to_ms: bool = False,  # pylint: disable=unused-argument
     include_all_columns: bool = False,
     sql_transpilation_library: Optional[str] = None,
+    use_materialized: bool = True,
 ) -> List[GeneratedSQL]:
     """
     Builds the measures SQL for a set of metrics with dimensions and filters.
@@ -158,7 +159,11 @@ async def get_measures_query(  # pylint: disable=too-many-locals
     measures_queries = []
     for parent_node, _ in common_parents.items():  # type: ignore
         measure_columns, dimensional_columns = [], []
-        query_builder = await QueryBuilder.create(session, parent_node.current)
+        query_builder = await QueryBuilder.create(
+            session,
+            parent_node.current,
+            use_materialized=use_materialized,
+        )
         parent_ast = await (
             query_builder.ignore_errors()
             .with_access_control(access_control)
@@ -233,9 +238,15 @@ class QueryBuilder:  # pylint: disable=too-many-instance-attributes,too-many-pub
     validation, allowing for dynamic node query generation based on runtime conditions.
     """
 
-    def __init__(self, session: AsyncSession, node_revision: NodeRevision):
+    def __init__(
+        self,
+        session: AsyncSession,
+        node_revision: NodeRevision,
+        use_materialized: bool = True,
+    ):
         self.session = session
         self.node_revision = node_revision
+        self.use_materialized = use_materialized
 
         self._filters: List[str] = []
         self._required_dimensions: List[str] = [
@@ -262,12 +273,13 @@ class QueryBuilder:  # pylint: disable=too-many-instance-attributes,too-many-pub
         cls,
         session: AsyncSession,
         node_revision: NodeRevision,
+        use_materialized: bool = True,
     ) -> "QueryBuilder":
         """
         Create a QueryBuilder instance for the node revision.
         """
         await session.refresh(node_revision, ["required_dimensions", "dimension_links"])
-        instance = cls(session, node_revision)
+        instance = cls(session, node_revision, use_materialized=use_materialized)
         return instance
 
     def ignore_errors(self):
@@ -464,6 +476,7 @@ class QueryBuilder:  # pylint: disable=too-many-instance-attributes,too-many-pub
             filters=self._filters,
             build_criteria=self._build_criteria,
             ctes_mapping=self.cte_mapping,
+            use_materialized=self.use_materialized,
         )
 
     def initialize_final_query_ast(self, node_ast, node_alias):
@@ -523,6 +536,7 @@ class QueryBuilder:  # pylint: disable=too-many-instance-attributes,too-many-pub
                     link,
                     self._filters,
                     self.cte_mapping,
+                    use_materialized=self.use_materialized,
                 )
                 dimension_join.node_query = convert_to_cte(
                     dimension_node_query,
@@ -751,6 +765,7 @@ async def build_dimension_node_query(
     link: DimensionLink,
     filters: List[str],
     cte_mapping: Dict[str, ast.Query],
+    use_materialized: bool = True,
 ):
     """
     Builds a dimension node query with the requested filters
@@ -778,6 +793,7 @@ async def build_dimension_node_query(
         filters=filters,  # type: ignore
         build_criteria=build_criteria,
         ctes_mapping=cte_mapping,
+        use_materialized=use_materialized,
     )
     return dimension_node_query
 
@@ -949,6 +965,7 @@ async def build_ast(  # pylint: disable=too-many-arguments,too-many-locals
     build_criteria: Optional[BuildCriteria] = None,
     access_control=None,
     ctes_mapping: Dict[str, ast.Query] = None,
+    use_materialized: bool = True,
 ) -> ast.Query:
     """
     Recursively replaces DJ node references with query ASTs. These are replaced with
@@ -974,14 +991,19 @@ async def build_ast(  # pylint: disable=too-many-arguments,too-many-locals
         for ref_expr in reference_expressions:
 
             # Try to find a materialized table attached to this node, if one exists.
-            physical_table = cast(
-                Optional[ast.Table],
-                get_table_for_node(
-                    referenced_node,
-                    build_criteria=build_criteria,
-                ),
-            )
+            physical_table = None
+            if use_materialized:
+                logger.debug("Checking for physical node: %s", referenced_node.name)
+                physical_table = cast(
+                    Optional[ast.Table],
+                    get_table_for_node(
+                        referenced_node,
+                        build_criteria=build_criteria,
+                    ),
+                )
+
             if not physical_table:
+                logger.debug("Didn't find physical node: %s", referenced_node.name)
                 # Build a new CTE with the query AST if there is no materialized table
                 if referenced_node.name not in ctes_mapping:
                     node_query = parse(cast(str, referenced_node.query))
@@ -994,6 +1016,7 @@ async def build_ast(  # pylint: disable=too-many-arguments,too-many-locals
                         build_criteria=build_criteria,
                         access_control=access_control,
                         ctes_mapping=ctes_mapping,
+                        use_materialized=use_materialized,
                     )
                     cte_name = ast.Name(amenable_name(referenced_node.name))
                     query_ast = query_ast.to_cte(cte_name, parent_ast=query)

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/datajunction_server/internal/nodes.py

@@ -83,10 +83,12 @@ from datajunction_server.sql.parsing import ast
 from datajunction_server.sql.parsing.ast import CompileContext
 from datajunction_server.sql.parsing.backends.antlr4 import parse
 from datajunction_server.typing import UTCDatetime
-from datajunction_server.utils import SEPARATOR, Version, VersionUpgrade
+from datajunction_server.utils import SEPARATOR, Version, VersionUpgrade, get_settings
 
 _logger = logging.getLogger(__name__)
 
+settings = get_settings()
+
 
 def get_node_column(node: Node, column_name: str) -> Column:
     """
@@ -260,7 +262,7 @@ async def create_node_revision(
         raise DJException(
             f"Cannot create nodes with multi-catalog dependencies: {set(catalog_ids)}",
         )
-    catalog_id = next(iter(catalog_ids), 0)
+    catalog_id = next(iter(catalog_ids), settings.default_catalog_id)
     parent_refs = (
         (
             await session.execute(

{datajunction_server-0.0.1a64.dev1 → datajunction_server-0.0.1a64.dev3}/tests/construction/build_v2_test.py

@@ -35,6 +35,7 @@ async def create_source(
     table: str,
     columns: List[Column],
     current_user: User,
+    query: str = None,
 ) -> Tuple[Node, NodeRevision]:
     """Create source node."""
     source_node = Node(
@@ -50,6 +51,7 @@ async def create_source(
         display_name=display_name,
         type=NodeType.SOURCE,
         version="1",
+        query=query,
         schema_=schema_,
         table=table,
         columns=columns,
@@ -136,7 +138,12 @@ async def events(session: AsyncSession, current_user: User) -> Node:
             Column(name="utc_date", type=ct.BigIntType(), order=4),
         ],
         current_user=current_user,
+        query=(
+            "SELECT event_id, user_id, device_id, country_code, "
+            "latency, utc_date FROM test.events"
+        ),
     )
+
     return events_node
 
 
@@ -262,6 +269,7 @@ async def devices(
             Column(name="device_manufacturer", type=ct.StringType(), order=2),
         ],
         current_user=current_user,
+        query="SELECT device_id, device_name, device_manufacturer FROM test.devices",
     )
 
     devices_dim_node, _ = await create_node_with_query(
@@ -312,6 +320,7 @@ async def manufacturers_dim(
             Column(name="created_on", type=ct.TimestampType(), order=2),
         ],
         current_user=current_user,
+        query="SELECT manufacturer_name, company_name, created_on FROM test.manufacturers",
     )
     manufacturers_dim_node, _ = await create_node_with_query(
         session,
@@ -365,6 +374,7 @@ async def country_dim(
             Column(name="population", type=ct.IntegerType(), order=3),
         ],
         current_user=current_user,
+        query="SELECT country_code, country_name, region_code, population FROM test.countries",
     )
 
     await create_source(
@@ -378,6 +388,7 @@ async def country_dim(
             Column(name="region_name", type=ct.StringType(), order=1),
         ],
         current_user=current_user,
+        query="SELECT region_code, region_name FROM test.regions",
     )
 
     await create_node_with_query(
@@ -1324,6 +1335,138 @@ async def test_query_builder(
     assert not query_builder.include_dimensions_in_groupby
 
 
+@pytest.mark.asyncio
+async def test_build_transform_sql_without_materialized_tables(
+    session: AsyncSession,
+    events: Node,  # pylint: disable=unused-argument
+    events_agg: Node,
+    devices: Node,  # pylint: disable=unused-argument
+    events_agg_devices_link: DimensionLink,  # pylint: disable=unused-argument
+    manufacturers_dim: Node,  # pylint: disable=unused-argument
+    country_dim: Node,  # pylint: disable=unused-argument
+    events_agg_countries_link: DimensionLink,  # pylint: disable=unused-argument
+):
+    """
+    Test building a transform node with filters and dimensions that forces skipping the materialized
+    tables for the dependent nodes.
+    """
+    query_builder = await QueryBuilder.create(
+        session,
+        events_agg.current,
+        use_materialized=False,
+    )
+    query_ast = await (
+        query_builder.filter_by("shared.manufacturers.company_name = 'Apple'")
+        .filter_by("shared.manufacturers.created_at > 20240101")
+        .filter_by("shared.countries.region_name = 'APAC'")
+        .add_dimension("shared.devices.device_manufacturer")
+        .add_dimension("shared.countries.region_name")
+        .build()
+    )
+    expected = """
+    WITH
+    source_DOT_events AS (
+    SELECT  event_id,
+        user_id,
+        device_id,
+        country_code,
+        latency,
+        utc_date
+     FROM test.events
+    ),
+    agg_DOT_events AS (
+    SELECT  source_DOT_events.user_id,
+        source_DOT_events.utc_date,
+        source_DOT_events.device_id,
+        source_DOT_events.country_code,
+        SUM(source_DOT_events.latency) AS total_latency
+     FROM source_DOT_events
+     GROUP BY  source_DOT_events.user_id, source_DOT_events.device_id, source_DOT_events.country_code
+    ),
+    source_DOT_devices AS (
+    SELECT  device_id,
+        device_name,
+        device_manufacturer
+     FROM test.devices
+    ),
+    shared_DOT_devices AS (
+    SELECT  CAST(source_DOT_devices.device_id AS INT) device_id,
+        CAST(source_DOT_devices.device_name AS STRING) device_name,
+        source_DOT_devices.device_manufacturer
+     FROM source_DOT_devices
+    ),
+    source_DOT_countries AS (
+    SELECT  country_code,
+        country_name,
+        region_code,
+        population
+     FROM test.countries
+    ),
+    source_DOT_regions AS (
+    SELECT  region_code,
+        region_name
+     FROM test.regions
+    ),
+    shared_DOT_regions AS (
+    SELECT  source_DOT_regions.region_code,
+        source_DOT_regions.region_name
+     FROM source_DOT_regions
+    ),
+    shared_DOT_countries AS (
+    SELECT  countries.country_code,
+        countries.country_name,
+        shared_DOT_regions.region_code,
+        shared_DOT_regions.region_name,
+        countries.population
+     FROM source_DOT_countries countries JOIN shared_DOT_regions ON countries.region_code = shared_DOT_regions.region_code
+     WHERE  shared_DOT_regions.region_name = 'APAC'
+    ),
+    source_DOT_manufacturers AS (
+    SELECT  manufacturer_name,
+        company_name,
+        created_on
+     FROM test.manufacturers
+    ),
+    shared_DOT_manufacturers AS (
+    SELECT  CAST(manufacturers.manufacturer_name AS STRING) name,
+        CAST(manufacturers.company_name AS STRING) company_name,
+        manufacturers.created_on AS created_at,
+        COUNT( DISTINCT devices.device_id) AS devices_produced
+     FROM source_DOT_manufacturers manufacturers JOIN shared_DOT_devices devices ON manufacturers.manufacturer_name = devices.device_manufacturer
+     WHERE  CAST(manufacturers.company_name AS STRING) = 'Apple' AND manufacturers.created_on > 20240101
+    )
+
+    SELECT  agg_DOT_events.user_id,
+        agg_DOT_events.utc_date,
+        agg_DOT_events.device_id,
+        agg_DOT_events.country_code,
+        agg_DOT_events.total_latency,
+        shared_DOT_devices.device_manufacturer shared_DOT_devices_DOT_device_manufacturer,
+        shared_DOT_countries.region_name shared_DOT_countries_DOT_region_name,
+        shared_DOT_manufacturers.company_name shared_DOT_manufacturers_DOT_company_name,
+        shared_DOT_manufacturers.created_at shared_DOT_manufacturers_DOT_created_at
+     FROM agg_DOT_events INNER JOIN shared_DOT_devices ON shared_DOT_devices.device_id = agg_DOT_events.device_id
+    INNER JOIN shared_DOT_countries ON agg_DOT_events.country_code = shared_DOT_countries.country_code
+    INNER JOIN shared_DOT_manufacturers ON shared_DOT_manufacturers.name = shared_DOT_devices.device_manufacturer
+    """
+    assert str(query_ast).strip() == str(parse(expected)).strip()
+
+    query_builder = await QueryBuilder.create(
+        session,
+        events_agg.current,
+        use_materialized=True,
+    )
+    query_ast = await (
+        query_builder.filter_by("shared.manufacturers.company_name = 'Apple'")
+        .filter_by("shared.manufacturers.created_at > 20240101")
+        .filter_by("shared.countries.region_name = 'APAC'")
+        .add_dimension("shared.devices.device_manufacturer")
+        .add_dimension("shared.countries.region_name")
+        .build()
+    )
+    assert str(query_ast).strip() != str(parse(expected)).strip()
+
+
 @pytest.mark.asyncio
 async def test_build_transform_with_multijoin_dimensions_with_extra_ctes(
     session: AsyncSession,

datajunction_server-0.0.1a64.dev1/datajunction_server/__about__.py

@@ -1,4 +0,0 @@
-"""
-Version for Hatch
-"""
-__version__ = "0.0.1a64.dev1"