datajunction-server 0.0.1a45.dev12__tar.gz → 0.0.1a45.dev14__tar.gz

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-server
-Version: 0.0.1a45.dev12
+Version: 0.0.1a45.dev14
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a45.dev14/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a45.dev14"

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/api/cubes.py

@@ -3,9 +3,9 @@
 Cube related APIs.
 """
 import logging
-from typing import List, Optional
+from typing import Annotated, List, Optional
 
-from fastapi import Depends, Query
+from fastapi import Depends, Header, Query
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from datajunction_server.api.helpers import get_catalog_by_name
@@ -98,6 +98,7 @@ async def get_cube_dimension_values(  # pylint: disable=too-many-locals
     ),
     include_counts: bool = False,
     async_: bool = False,
+    cache_control: Annotated[str, Header()] = "",
     session: AsyncSession = Depends(get_session),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=redefined-outer-name
@@ -132,7 +133,10 @@ async def get_cube_dimension_values(  # pylint: disable=too-many-locals
         submitted_query=translated_sql.sql,
         async_=async_,
     )
-    result = query_service_client.submit_query(query_create)
+    result = query_service_client.submit_query(
+        query_create,
+        headers={"Cache-Control": cache_control},
+    )
     count_column = [
         idx
         for idx, col in enumerate(translated_sql.columns)  # type: ignore

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/api/data.py

@@ -3,9 +3,9 @@
 Data related APIs.
 """
 from http import HTTPStatus
-from typing import Dict, List, Optional
+from typing import Annotated, Dict, List, Optional
 
-from fastapi import Depends, Query, Request
+from fastapi import Depends, Header, Query, Request
 from fastapi.responses import JSONResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, selectinload
@@ -175,6 +175,7 @@ async def get_data(  # pylint: disable=too-many-locals
         default=False,
         description="Whether to run the query async or wait for results from the query engine",
     ),
+    cache_control: Annotated[str, Header()] = "",
     session: AsyncSession = Depends(get_session),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     engine_name: Optional[str] = None,
@@ -235,7 +236,10 @@ async def get_data(  # pylint: disable=too-many-locals
         submitted_query=query.sql,
         async_=async_,
     )
-    result = query_service_client.submit_query(query_create)
+    result = query_service_client.submit_query(
+        query_create,
+        headers={"Cache-Control": cache_control},
+    )
     # Inject column info if there are results
     if result.results.__root__:  # pragma: no cover
         result.results.__root__[0].columns = columns
@@ -273,6 +277,7 @@ async def get_data_for_metrics(  # pylint: disable=R0914, R0913
     limit: Optional[int] = None,
     async_: bool = False,
     *,
+    cache_control: Annotated[str, Header()] = "",
     session: AsyncSession = Depends(get_session),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     engine_name: Optional[str] = None,
@@ -310,7 +315,10 @@ async def get_data_for_metrics(  # pylint: disable=R0914, R0913
         submitted_query=translated_sql.sql,
         async_=async_,
     )
-    result = query_service_client.submit_query(query_create)
+    result = query_service_client.submit_query(
+        query_create,
+        headers={"Cache-Control": cache_control},
+    )
 
     # Inject column info if there are results
     if result.results.__root__:  # pragma: no cover
@@ -326,6 +334,7 @@ async def get_data_stream_for_metrics(  # pylint: disable=R0914, R0913
     orderby: List[str] = Query([]),
     limit: Optional[int] = None,
     *,
+    cache_control: Annotated[str, Header()] = "",
     session: AsyncSession = Depends(get_session),
     request: Request,
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
@@ -354,7 +363,10 @@ async def get_data_stream_for_metrics(  # pylint: disable=R0914, R0913
         async_=True,
     )
     # Submits the query, equivalent to calling POST /data/ directly
-    initial_query_info = query_service_client.submit_query(query_create)
+    initial_query_info = query_service_client.submit_query(
+        query_create,
+        headers={"Cache-Control": cache_control},
+    )
     return EventSourceResponse(
         query_event_stream(
             query=initial_query_info,

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/api/djsql.py

@@ -2,9 +2,9 @@
 Data related APIs.
 """
 
-from typing import Optional
+from typing import Annotated, Optional
 
-from fastapi import Depends, Request
+from fastapi import Depends, Header, Request
 from sqlalchemy.ext.asyncio import AsyncSession
 from sse_starlette.sse import EventSourceResponse
 
@@ -31,6 +31,7 @@ async def get_data_for_djsql(  # pylint: disable=R0914, R0913
     query: str,
     async_: bool = False,
     *,
+    cache_control: Annotated[str, Header()] = "",
     session: AsyncSession = Depends(get_session),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     engine_name: Optional[str] = None,
@@ -64,7 +65,10 @@ async def get_data_for_djsql(  # pylint: disable=R0914, R0913
         async_=async_,
     )
 
-    result = query_service_client.submit_query(query_create)
+    result = query_service_client.submit_query(
+        query_create,
+        headers={"Cache-Control": cache_control},
+    )
 
     # Inject column info if there are results
     if result.results.__root__:  # pragma: no cover
@@ -77,6 +81,7 @@ async def get_data_for_djsql(  # pylint: disable=R0914, R0913
 async def get_data_stream_for_djsql(  # pragma: no cover
     query: str,
     *,
+    cache_control: Annotated[str, Header()] = "",
     session: AsyncSession = Depends(get_session),
     request: Request,
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
@@ -111,7 +116,10 @@ async def get_data_stream_for_djsql(  # pragma: no cover
     )
 
     # Submits the query, equivalent to calling POST /data/ directly
-    initial_query_info = query_service_client.submit_query(query_create)
+    initial_query_info = query_service_client.submit_query(
+        query_create,
+        headers={"Cache-Control": cache_control},
+    )
     return EventSourceResponse(
         await query_event_stream(
             query=initial_query_info,

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/api/helpers.py

@@ -512,14 +512,16 @@ async def resolve_downstream_references(
                     await session.execute(
                         select(NodeRevision)
                         .where(NodeRevision.id == downstream_node_id)
-                        .options(joinedload(NodeRevision.missing_parents)),
+                        .options(
+                            joinedload(NodeRevision.missing_parents),
+                            joinedload(NodeRevision.parents),
+                        ),
                     )
                 )
                 .unique()
                 .scalar_one()
             )
             await session.refresh(node_revision, ["node"])
-            await session.refresh(downstream_node_revision, ["parents", "missing_parents"])
             downstream_node_revision.parents.append(node_revision.node)
             downstream_node_revision.missing_parents.remove(missing_parent)
             node_validator = await validate_node_data(
@@ -631,6 +633,25 @@ async def validate_cube(  # pylint: disable=too-many-locals
         ),
         key=lambda x: metrics_sorting_order.get(x.name, 0),
     )
+
+    # Verify that all metrics exist
+    if len(metric_nodes) != len(metric_names):
+        not_found = set(metric_names) - {metric.name for metric in metric_nodes}
+        raise DJNodeNotFound(
+            f"The following metric nodes were not found: {', '.join(not_found)}",
+        )
+
+    # Verify that all metrics are in valid status
+    invalid_metrics = [
+        metric.name
+        for metric in metric_nodes
+        if metric.current.status == NodeStatus.INVALID
+    ]
+    if invalid_metrics:
+        raise DJInvalidInputException(
+            f"The following metric nodes are invalid: {', '.join(invalid_metrics)}",
+        )
+
     metrics: List[Column] = [metric.current.columns[0] for metric in metric_nodes]
     catalogs = [metric.current.catalog for metric in metric_nodes]
     catalog = catalogs[0] if catalogs else None
@@ -744,7 +765,8 @@ async def get_history(
                 .where(History.entity_type == entity_type)
                 .where(History.entity_name == entity_name)
                 .offset(offset)
-                .limit(limit),
+                .limit(limit)
+                .order_by(History.created_at.desc()),
             )
         )
         .scalars()

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/api/history.py

@@ -58,7 +58,7 @@ async def list_history_by_node_context(
             await session.execute(
                 select(History)
                 .where(History.node == node)
-                .order_by(History.created_at)
+                .order_by(History.created_at.desc())
                 .offset(offset)
                 .limit(limit),
             )

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/construction/build.py

@@ -437,29 +437,20 @@ async def _build_tables_on_select(
                             for link in node.dimension_links
                             for left, right in link.foreign_key_mapping().items()
                         }
-                        foreign_keys_alias_map = {
-                            col.alias_or_name.name: col  # type: ignore
-                            for col in node_query.select.projection
-                        }
                         if all(
                             col.alias_or_name.name in fk_column_mapping
                             or col.alias_or_name.name in foreign_keys_map
                             for col in referenced_cols
                         ):
-                            # Renames the columns to the foreign key columns
                             for col in referenced_cols:
                                 ref_col_name = (
                                     fk_column_mapping[col.alias_or_name.name].name
                                     if col.alias_or_name.name in fk_column_mapping
                                     else foreign_keys_map[
                                         col.alias_or_name.name
-                                    ].alias_or_name
+                                    ].alias_or_name.name
                                 )
                                 col.name = ast.Name(name=ref_col_name)
-                                if col.alias_or_name.name in foreign_keys_alias_map:
-                                    col.name = foreign_keys_alias_map[  # type: ignore
-                                        col.alias_or_name.name
-                                    ].name
                             filter_asts.append(
                                 temp_select.where,  # type:ignore
                             )

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/errors.py

@@ -32,6 +32,7 @@ class ErrorCode(IntEnum):
     NODE_TYPE_ERROR = 204
     INVALID_DIMENSION_JOIN = 205
     INVALID_COLUMN = 206
+    QUERY_SERVICE_ERROR = 207
 
     # SQL Build Error
     COMPOUND_BUILD_EXCEPTION = 300

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/internal/nodes.py

@@ -865,19 +865,20 @@ async def propagate_update_downstream(  # pylint: disable=too-many-locals
                         ),
                     ],
                 )
-                await update_cube_node(
-                    session,
-                    child_node.current,  # type: ignore
-                    UpdateNode(
-                        metrics=[
-                            metric.name for metric in child_node.current.cube_metrics()  # type: ignore  # pylint: disable=line-too-long
-                        ],
-                        dimensions=child_node.current.cube_dimensions(),  # type: ignore
-                    ),
-                    query_service_client=query_service_client,
-                    background_tasks=background_tasks,
-                    validate_access=validate_access,
-                )
+                if child_node:
+                    await update_cube_node(
+                        session,
+                        child_node.current,  # type: ignore
+                        UpdateNode(
+                            metrics=[
+                                metric.name for metric in child_node.current.cube_metrics()  # type: ignore  # pylint: disable=line-too-long
+                            ],
+                            dimensions=child_node.current.cube_dimensions(),  # type: ignore
+                        ),
+                        query_service_client=query_service_client,
+                        background_tasks=background_tasks,
+                        validate_access=validate_access,
+                    )
                 continue
 
             node_validator = await validate_node_data(
@@ -893,6 +894,8 @@ async def propagate_update_downstream(  # pylint: disable=too-many-locals
                     ),
                 ],
             )
+            if not result:
+                continue
             child = result.current  # type: ignore
             # Update the child with a new node revision if its columns or status have changed
             if node_validator.differs_from(child):
@@ -982,16 +985,6 @@ def copy_existing_node_revision(old_revision: NodeRevision):
         status=old_revision.status,
         required_dimensions=old_revision.required_dimensions,
         metric_metadata=old_revision.metric_metadata,
-        dimension_links=[
-            DimensionLink(
-                dimension_id=link.dimension_id,
-                join_sql=link.join_sql,
-                join_type=link.join_type,
-                join_cardinality=link.join_cardinality,
-                materialization_conf=link.materialization_conf,
-            )
-            for link in old_revision.dimension_links
-        ],
     )
 
 

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/models/metric.py

@@ -13,7 +13,7 @@ from datajunction_server.models.node import (
     MetricMetadataOutput,
 )
 from datajunction_server.models.query import ColumnMetadata
-from datajunction_server.sql.parsing.backends.antlr4 import parse
+from datajunction_server.sql.parsing.backends.antlr4 import ast, parse
 from datajunction_server.transpilation import get_transpilation_plugin
 from datajunction_server.typing import UTCDatetime
 from datajunction_server.utils import get_settings
@@ -41,12 +41,20 @@ class Metric(BaseModel):
     metric_metadata: Optional[MetricMetadataOutput] = None
     required_dimensions: List[str]
 
+    incompatible_druid_functions: List[str]
+
     @classmethod
     def parse_node(cls, node: Node, dims: List[DimensionAttributeOutput]) -> "Metric":
         """
         Parses a node into a metric.
         """
         query_ast = parse(node.current.query)
+        functions = [func.function() for func in query_ast.find_all(ast.Function)]
+        incompatible_druid_functions = [
+            func.__name__.upper()
+            for func in functions
+            if Dialect.DRUID not in func.dialects
+        ]
         return cls(
             id=node.id,
             name=node.name,
@@ -61,6 +69,7 @@ class Metric(BaseModel):
             dimensions=dims,
             metric_metadata=node.current.metric_metadata,
             required_dimensions=[dim.name for dim in node.current.required_dimensions],
+            incompatible_druid_functions=incompatible_druid_functions,
         )
 
 

{datajunction_server-0.0.1a45.dev12 → datajunction_server-0.0.1a45.dev14}/datajunction_server/service_clients.py

@@ -1,6 +1,6 @@
 """Clients for various configurable services."""
 from http import HTTPStatus
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Union
 from urllib.parse import urljoin
 
 import requests
@@ -10,7 +10,9 @@ from urllib3 import Retry
 from datajunction_server.database.column import Column
 from datajunction_server.errors import (
     DJDoesNotExistException,
+    DJError,
     DJQueryServiceClientException,
+    ErrorCode,
 )
 from datajunction_server.models.materialization import (
     DruidMaterializationInput,
@@ -120,18 +122,27 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
     def submit_query(  # pylint: disable=too-many-arguments
         self,
         query_create: QueryCreate,
+        headers: Optional[Dict[str, str]] = None,
     ) -> QueryWithResults:
         """
         Submit a query to the query service
         """
+        if not headers:
+            headers = {"Cache-Control": ""}
         response = self.requests_session.post(
             "/queries/",
+            headers=headers,
             json=query_create.dict(),
         )
         response_data = response.json()
         if response.status_code not in (200, 201):
             raise DJQueryServiceClientException(
                 message=f"Error response from query service: {response_data['message']}",
+                errors=[
+                    DJError(code=ErrorCode.QUERY_SERVICE_ERROR, message=error)
+                    for error in response_data["errors"]
+                ],
+                http_status_code=response.status_code,
             )
         query_info = response.json()
         return QueryWithResults(**query_info)