datajunction-server 0.0.1a63.dev0__tar.gz → 0.0.1a64.dev0__tar.gz

{datajunction_server-0.0.1a63.dev0 → datajunction_server-0.0.1a64.dev0}/.coveragerc

@@ -9,6 +9,8 @@ omit =
     */datajunction_server/sql/parsing/ast.py
     */datajunction_server/internal/access/authentication/google.py
     */datajunction_server/api/access/authentication/google.py
+    */datajunction_server/api/djsql.py
+    */datajunction_server/construction/dj_query.py
 
 [paths]
 source =

{datajunction_server-0.0.1a63.dev0 → datajunction_server-0.0.1a64.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a63.dev0
+Version: 0.0.1a64.dev0
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a64.dev0/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a64.dev0"

{datajunction_server-0.0.1a63.dev0 → datajunction_server-0.0.1a64.dev0}/datajunction_server/api/data.py

@@ -203,11 +203,15 @@ async def get_data(  # pylint: disable=too-many-locals
     node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)
     available_engines = node.current.catalog.engines  # type: ignore
     engine = (
-        await get_engine(session, engine_name, engine_version)  # type: ignore
-        if engine_name
+        await get_engine(
+            session,
+            engine_name or query_request.engine_name,
+            engine_version or query_request.engine_version,  # type: ignore
+        )
+        if engine_name or query_request.engine_name
         else available_engines[0]
     )
-    if engine not in available_engines:
+    if engine not in available_engines and engine.name != query_request.engine_name:
         raise DJInvalidInputException(  # pragma: no cover
             f"The selected engine is not available for the node {node_name}. "
             f"Available engines include: {', '.join(engine.name for engine in available_engines)}",
@@ -427,11 +431,21 @@ async def get_data_stream_for_metrics(  # pylint: disable=R0914, R0913
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     engine_name: Optional[str] = None,
     engine_version: Optional[str] = None,
+    current_user: User = Depends(get_and_update_current_user),
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> QueryWithResults:
     """
     Return data for a set of metrics with dimensions and filters using server sent events
     """
     request_headers = dict(request.headers)
+    access_control = access.AccessControlStore(
+        validate_access=validate_access,
+        user=current_user,
+        base_verb=access.ResourceRequestVerb.READ,
+    )
+
     translated_sql, engine, catalog = await build_sql_for_multiple_metrics(
         session,
         metrics,
@@ -441,6 +455,7 @@ async def get_data_stream_for_metrics(  # pylint: disable=R0914, R0913
         limit,
         engine_name,
         engine_version,
+        access_control,
     )
 
     query_create = QueryCreate(

{datajunction_server-0.0.1a63.dev0 → datajunction_server-0.0.1a64.dev0}/datajunction_server/api/graphql/scalars/node.py

@@ -77,7 +77,7 @@ class Tag:  # pylint: disable=too-few-public-methods
     """
 
     name: str
-    description: str = ""
+    description: Optional[str]
     tag_type: str
     tag_metadata: JSON
 

{datajunction_server-0.0.1a63.dev0 → datajunction_server-0.0.1a64.dev0}/datajunction_server/api/helpers.py

@@ -20,7 +20,6 @@ from sqlalchemy.sql.operators import and_, is_
 from datajunction_server.construction.build import (
     build_materialized_cube_node,
     build_metric_nodes,
-    build_node,
     get_default_criteria,
     rename_columns,
     validate_shared_dimensions,
@@ -203,17 +202,22 @@ async def get_query(  # pylint: disable=too-many-arguments
     """
     Get a query for a metric, dimensions, and filters
     """
-    node = await Node.get_by_name(session, node_name)
+    from datajunction_server.construction.build_v2 import (  # pylint: disable=import-outside-toplevel
+        QueryBuilder,
+    )
+
+    node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)
     build_criteria = get_default_criteria(node.current, engine)  # type: ignore
-    query_ast = await build_node(
-        session=session,
-        node=node.current,  # type: ignore
-        filters=filters,
-        dimensions=dimensions,
-        orderby=orderby,
-        limit=limit,
-        build_criteria=build_criteria,
-        access_control=access_control,
+    query_builder = await QueryBuilder.create(session, node.current)  # type: ignore
+    query_ast = await (
+        query_builder.ignore_errors()
+        .with_access_control(access_control)
+        .with_build_criteria(build_criteria)
+        .add_dimensions(dimensions)
+        .add_filters(filters)
+        .limit(limit)
+        .order_by(orderby)
+        .build()
     )
     query_ast = rename_columns(query_ast, node.current)  # type: ignore
     return query_ast
@@ -369,7 +373,6 @@ async def validate_cube(  # pylint: disable=too-many-locals
         ),
         key=lambda x: metrics_sorting_order.get(x.name, 0),
     )
-
     # Verify that all metrics exist
     if len(metric_nodes) != len(metric_names):
         not_found = set(metric_names) - {metric.name for metric in metric_nodes}
@@ -493,7 +496,6 @@ async def validate_cube(  # pylint: disable=too-many-locals
         session,
         metric_nodes,
         dimension_names,
-        [],
     )
     return metrics, metric_nodes, list(dimension_nodes.values()), dimensions, catalog
 
@@ -596,6 +598,7 @@ async def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,
     engine_version: Optional[str] = None,
     access_control: Optional[access.AccessControlStore] = None,
     use_materialized: bool = True,
+    ignore_errors: bool = True,
 ) -> Tuple[TranslatedSQL, Engine, Catalog]:
     """
     Build SQL for multiple metrics. Used by both /sql and /data endpoints
@@ -620,7 +623,13 @@ async def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,
             ),
         ],
     )
-    available_engines = leading_metric_node.current.catalog.engines  # type: ignore
+    if access_control:
+        access_control.add_request_by_node(leading_metric_node.current)  # type: ignore
+    available_engines = (
+        leading_metric_node.current.catalog.engines  # type: ignore
+        if leading_metric_node.current.catalog  # type: ignore
+        else []
+    )
 
     # Try to find a built cube that already has the given metrics and dimensions
     # The cube needs to have a materialization configured and an availability state
@@ -687,6 +696,7 @@ async def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,
             )
             for col in dimension_columns
         ]
+        engine = materialized_cube_catalog.engines[0]
         return (
             TranslatedSQL(
                 sql=str(query_ast),
@@ -705,6 +715,7 @@ async def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,
         orderby=orderby or [],
         limit=limit,
         access_control=access_control,
+        ignore_errors=ignore_errors,
     )
     columns = [
         assemble_column_metadata(col)  # type: ignore
@@ -796,7 +807,7 @@ async def query_event_stream(  # pylint: disable=too-many-arguments
         await asyncio.sleep(stream_delay)  # pragma: no cover
 
 
-async def build_sql_for_dj_query(  # pylint: disable=too-many-arguments,too-many-locals
+async def build_sql_for_dj_query(  # pylint: disable=too-many-locals  # pragma: no cover
     session: AsyncSession,
     query: str,
     access_control: access.AccessControl,
@@ -809,35 +820,39 @@ async def build_sql_for_dj_query(  # pylint: disable=too-many-arguments,too-many
 
     query_ast, dj_nodes = await build_dj_query(session, query)
 
-    for node in dj_nodes:
-        access_control.add_request_by_node(
+    for node in dj_nodes:  # pragma: no cover
+        access_control.add_request_by_node(  # pragma: no cover
             node.current,
         )
 
-    access_control.validate_and_raise()
+    access_control.validate_and_raise()  # pragma: no cover
 
-    leading_metric_node = dj_nodes[0]
-    available_engines = leading_metric_node.current.catalog.engines
+    leading_metric_node = dj_nodes[0]  # pragma: no cover
+    available_engines = (  # pragma: no cover
+        leading_metric_node.current.catalog.engines
+        if leading_metric_node.current.catalog
+        else []
+    )
 
     # Check if selected engine is available
-    engine = (
+    engine = (  # pragma: no cover
         await get_engine(session, engine_name, engine_version)  # type: ignore
         if engine_name
         else available_engines[0]
     )
 
-    if engine not in available_engines:
+    if engine not in available_engines:  # pragma: no cover
         raise DJInvalidInputException(  # pragma: no cover
             f"The selected engine is not available for the node {leading_metric_node.name}. "
             f"Available engines include: {', '.join(engine.name for engine in available_engines)}",
         )
 
-    columns = [
+    columns = [  # pragma: no cover
         ColumnMetadata(name=col.alias_or_name.name, type=str(col.type))  # type: ignore
         for col in query_ast.select.projection
     ]
 
-    return (
+    return (  # pragma: no cover
         TranslatedSQL(
             sql=str(query_ast),
             columns=columns,

{datajunction_server-0.0.1a63.dev0 → datajunction_server-0.0.1a64.dev0}/datajunction_server/api/sql.py

@@ -39,88 +39,6 @@ settings = get_settings()
 router = SecureAPIRouter(tags=["sql"])
 
 
-@router.get("/sql/measures/", response_model=TranslatedSQL, name="Get Measures SQL")
-async def get_measures_sql_for_cube(
-    metrics: List[str] = Query([]),
-    dimensions: List[str] = Query([]),
-    filters: List[str] = Query([]),
-    *,
-    include_all_columns: bool = Query(
-        False,
-        description=(
-            "Whether to include all columns or only those necessary "
-            "for the metrics and dimensions in the cube"
-        ),
-    ),
-    session: AsyncSession = Depends(get_session),
-    engine_name: Optional[str] = None,
-    engine_version: Optional[str] = None,
-    current_user: User = Depends(get_and_update_current_user),
-    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
-        validate_access,
-    ),
-) -> TranslatedSQL:
-    """
-    Return the measures SQL for a set of metrics with dimensions and filters.
-    This SQL can be used to produce an intermediate table with all the measures
-    and dimensions needed for an analytics database (e.g., Druid).
-    """
-    from datajunction_server.construction.build import (  # pylint: disable=import-outside-toplevel,line-too-long
-        get_measures_query,
-    )
-
-    if query_request := await QueryRequest.get_query_request(
-        session,
-        nodes=metrics,
-        dimensions=dimensions,
-        filters=filters,
-        orderby=[],
-        limit=None,
-        engine_name=engine_name,
-        engine_version=engine_version,
-        query_type=QueryBuildType.MEASURES,
-        other_args={"include_all_columns": include_all_columns},
-    ):
-        engine = (
-            await get_engine(session, engine_name, engine_version)  # type: ignore
-            if engine_name
-            else None
-        )
-        return TranslatedSQL(
-            sql=query_request.query,
-            columns=query_request.columns,
-            dialect=engine.dialect if engine else None,
-        )
-
-    measures_query = await get_measures_query(
-        session=session,
-        metrics=metrics,
-        dimensions=dimensions,
-        filters=filters,
-        engine_name=engine_name,
-        engine_version=engine_version,
-        current_user=current_user,
-        validate_access=validate_access,
-        include_all_columns=include_all_columns,
-    )
-
-    await QueryRequest.save_query_request(
-        session=session,
-        nodes=metrics,
-        dimensions=dimensions,
-        filters=filters,
-        orderby=[],
-        limit=None,
-        engine_name=engine_name,
-        engine_version=engine_version,
-        query_type=QueryBuildType.MEASURES,
-        query=measures_query.sql,
-        columns=[col.dict() for col in measures_query.columns],  # type: ignore
-        other_args={"include_all_columns": include_all_columns},
-    )
-    return measures_query
-
-
 @router.get(
     "/sql/measures/v2/",
     response_model=List[GeneratedSQL],
@@ -162,6 +80,7 @@ async def get_measures_sql_for_cube_v2(
         get_measures_query,
     )
 
+    metrics = list(OrderedDict.fromkeys(set(metrics)))
     measures_query = await get_measures_query(
         session=session,
         metrics=metrics,
@@ -187,6 +106,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
     session: AsyncSession = Depends(get_session),
     engine: Engine,
     access_control: AccessControlStore,
+    ignore_errors: bool = True,
 ) -> QueryRequest:
     """
     Build node SQL and save it to query requests
@@ -202,7 +122,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
         dimensions = list(
             OrderedDict.fromkeys(node.current.cube_node_dimensions + dimensions),
         )
-        translated_sql, _, _ = await build_sql_for_multiple_metrics(
+        translated_sql, engine, _ = await build_sql_for_multiple_metrics(
             session=session,
             metrics=node.current.cube_node_metrics,
             dimensions=dimensions,
@@ -236,21 +156,39 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
         return request
 
     # For all other nodes, build the node query
-    query_ast = await get_query(
-        session=session,
-        node_name=node_name,
-        dimensions=dimensions,
-        filters=filters,
-        orderby=orderby,
-        limit=limit,
-        engine=engine,
-        access_control=access_control,
-    )
-    columns = [
-        assemble_column_metadata(col)  # type: ignore
-        for col in query_ast.select.projection
-    ]
-    query = str(query_ast)
+    node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)  # type: ignore
+    if node.type == NodeType.METRIC:
+        translated_sql, engine, _ = await build_sql_for_multiple_metrics(
+            session,
+            [node_name],
+            dimensions,
+            filters,
+            orderby,
+            limit,
+            engine.name if engine else None,
+            engine.version if engine else None,
+            access_control=access_control,
+            ignore_errors=ignore_errors,
+        )
+        query = translated_sql.sql
+        columns = translated_sql.columns
+    else:
+        query_ast = await get_query(
+            session=session,
+            node_name=node_name,
+            dimensions=dimensions,
+            filters=filters,
+            orderby=orderby,
+            limit=limit,
+            engine=engine,
+            access_control=access_control,
+        )
+        columns = [
+            assemble_column_metadata(col)  # type: ignore
+            for col in query_ast.select.projection
+        ]
+        query = str(query_ast)
+
     query_request = await QueryRequest.save_query_request(
         session=session,
         nodes=[node_name],
@@ -262,7 +200,7 @@ async def build_and_save_node_sql(  # pylint: disable=too-many-locals
         engine_version=engine.version if engine else None,
         query_type=QueryBuildType.NODE,
         query=query,
-        columns=[col.dict() for col in columns],
+        columns=[col.dict() for col in columns or []],
     )
     return query_request
 
@@ -280,6 +218,7 @@ async def get_node_sql(  # pylint: disable=too-many-locals
     current_user: User,
     validate_access: access.ValidateAccessFn,  # pylint: disable=redefined-outer-name
     background_tasks: BackgroundTasks,
+    ignore_errors: bool = True,
 ) -> Tuple[TranslatedSQL, QueryRequest]:
     """
     Return SQL for a node.
@@ -339,6 +278,7 @@ async def get_node_sql(  # pylint: disable=too-many-locals
         session=session,
         engine=engine,  # type: ignore
         access_control=access_control,
+        ignore_errors=ignore_errors,
     )
     return (
         TranslatedSQL(
@@ -370,6 +310,7 @@ async def get_sql(  # pylint: disable=too-many-locals
         validate_access,
     ),
     background_tasks: BackgroundTasks,
+    ignore_errors: Optional[bool] = True,
 ) -> TranslatedSQL:
     """
     Return SQL for a node.
@@ -386,6 +327,7 @@ async def get_sql(  # pylint: disable=too-many-locals
         current_user=current_user,
         validate_access=validate_access,
         background_tasks=background_tasks,
+        ignore_errors=ignore_errors,  # type: ignore
     )
     return translated_sql
 
@@ -405,6 +347,7 @@ async def get_sql_for_metrics(
     validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
         validate_access,
     ),
+    ignore_errors: Optional[bool] = True,
 ) -> TranslatedSQL:
     """
     Return SQL for a set of metrics with dimensions and filters
@@ -448,6 +391,7 @@ async def get_sql_for_metrics(
         engine_name,
         engine_version,
         access_control,
+        ignore_errors=ignore_errors,  # type: ignore
     )
 
     await QueryRequest.save_query_request(