datajunction-server 0.0.1a84.dev4__tar.gz → 0.0.1a85.dev0__tar.gz

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a84.dev4
+Version: 0.0.1a85.dev0
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: Homepage, https://datajunction.io
 Project-URL: Repository, https://github.com/DataJunction/dj

datajunction_server-0.0.1a85.dev0/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a85.dev0"

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/api/cubes.py

@@ -12,6 +12,7 @@ from datajunction_server.api.helpers import get_catalog_by_name
 from datajunction_server.construction.dimensions import build_dimensions_from_cube_query
 from datajunction_server.database.node import Node
 from datajunction_server.database.user import User
+from datajunction_server.errors import DJInvalidInputException
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
 from datajunction_server.internal.access.authorization import validate_access
 from datajunction_server.internal.materializations import build_cube_materialization
@@ -57,7 +58,7 @@ async def get_cube(
     return await get_cube_revision_metadata(session, name)
 
 
-@router.get("/cubes/{name}/materialization", name="Materialization Config Cube")
+@router.get("/cubes/{name}/materialization", name="Cube Materialization Config")
 async def cube_materialization_info(
     name: str,
     session: AsyncSession = Depends(get_session),
@@ -88,8 +89,11 @@ async def cube_materialization_info(
     """
     node = await Node.get_cube_by_name(session, name)
     temporal_partitions = node.current.temporal_partition_columns()  # type: ignore
-    # if not temporal_partitions:
-    #     raise DJInvalidInputException()
+    if len(temporal_partitions) != 1:
+        raise DJInvalidInputException(
+            "The cube must have a single temporal partition column set "
+            "in order for it to be materialized.",
+        )
     temporal_partition = temporal_partitions[0] if temporal_partitions else None
     granularity_lookback_defaults = {
         Granularity.MINUTE: "1 MINUTE",

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/api/data.py

@@ -2,6 +2,7 @@
 """
 Data related APIs.
 """
+import logging
 from typing import Callable, Dict, List, Optional
 
 from fastapi import BackgroundTasks, Depends, Query, Request
@@ -42,6 +43,8 @@ from datajunction_server.utils import (
     get_settings,
 )
 
+_logger = logging.getLogger(__name__)
+
 settings = get_settings()
 router = SecureAPIRouter(tags=["data"])
 
@@ -61,6 +64,7 @@ async def add_availability_state(
     """
     Add an availability state to a node.
     """
+    _logger.info("Storing availability for node=%s", node_name)
 
     node = await Node.get_by_name(
         session,

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/api/helpers.py

@@ -366,6 +366,8 @@ async def validate_cube(  # pylint: disable=too-many-locals
 
     # Verify that the provided metrics are metric nodes
     metrics: List[Column] = [metric.current.columns[0] for metric in metric_nodes]
+    for metric in metrics:
+        await session.refresh(metric, ["node_revisions"])
     if not metrics:
         raise DJInvalidInputException(
             message=("At least one metric is required"),

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/api/materializations.py

@@ -81,7 +81,7 @@ def materialization_jobs_info() -> JSONResponse:
 )
 async def upsert_materialization(  # pylint: disable=too-many-locals
     node_name: str,
-    data: UpsertCubeMaterialization | UpsertMaterialization,
+    data: UpsertMaterialization | UpsertCubeMaterialization,
     *,
     session: AsyncSession = Depends(get_session),
     request: Request,
@@ -105,7 +105,7 @@ async def upsert_materialization(  # pylint: disable=too-many-locals
     if node.type == NodeType.CUBE:  # type: ignore
         node = await Node.get_cube_by_name(session, node_name)
     _logger.info(
-        "Upserting materialization for node=%s@%s",
+        "Upserting materialization for node=%s version=%s",
         node.name,  # type: ignore
         node.current_version,  # type: ignore
     )
@@ -138,7 +138,7 @@ async def upsert_materialization(  # pylint: disable=too-many-locals
         and existing_materialization.config == new_materialization.config
     ):
         _logger.info(
-            "Existing materialization found for node=%s@%s",
+            "Existing materialization found for node=%s version=%s",
             node.name,  # type: ignore
             node.current_version,  # type: ignore
         )
@@ -167,7 +167,7 @@ async def upsert_materialization(  # pylint: disable=too-many-locals
             request_headers=request_headers,
         )
         _logger.info(
-            "Refresh materialization workflows for node=%s@%s",
+            "Refresh materialization workflows for node=%s version=%s",
             node.name,  # type: ignore
             node.current_version,  # type: ignore
         )
@@ -195,7 +195,7 @@ async def upsert_materialization(  # pylint: disable=too-many-locals
     # If changes are detected, update the existing or save the new materialization
     if existing_materialization:
         _logger.info(
-            "Updating existing materialization for node=%s@%s",
+            "Updating existing materialization for node=%s version=%s",
             node.name,  # type: ignore
             node.current_version,  # type: ignore
         )
@@ -206,7 +206,7 @@ async def upsert_materialization(  # pylint: disable=too-many-locals
         new_materialization.deactivated_at = None
     else:
         _logger.info(
-            "Adding new materialization for node=%s@%s",
+            "Adding new materialization for node=%s version=%s",
             node.name,  # type: ignore
             node.current_version,  # type: ignore
         )
@@ -242,7 +242,7 @@ async def upsert_materialization(  # pylint: disable=too-many-locals
     )
     await session.commit()
     _logger.info(
-        "Scheduling materialization workflows for node=%s@%s",
+        "Scheduling materialization workflows for node=%s version=%s",
         node.name,  # type: ignore
         node.current_version,  # type: ignore
     )

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/internal/cube_materializations.py

@@ -37,9 +37,16 @@ def generate_partition_filter_sql(
         lookback_window == "1 DAY" or not lookback_window
     ):
         return f"{temporal_partition.name} = {partition_sql}"
-    lookback_timestamp = f"{logical_ts} - INTERVAL {lookback_window}"
-    partition_start = _partition_sql(lookback_timestamp, str(temporal_partition.type))
-    return f"{temporal_partition.name} BETWEEN {partition_start} AND {partition_sql}"
+    lookback_timestamp = (
+        f"{logical_ts} - INTERVAL {lookback_window}"  # pragma: no cover
+    )
+    partition_start = _partition_sql(  # pragma: no cover
+        lookback_timestamp,
+        str(temporal_partition.type),
+    )
+    return (  # pragma: no cover
+        f"{temporal_partition.name} BETWEEN {partition_start} AND {partition_sql}"
+    )
 
 
 def combine_measures_on_shared_grain(
@@ -161,7 +168,7 @@ async def build_cube_materialization(  # pylint: disable=used-before-assignment,
         )
     }
     if len(query_grains) > 1:
-        raise DJInvalidInputException(
+        raise DJInvalidInputException(  # pragma: no cover
             "DJ cannot manage materializations for cubes that have underlying "
             "measures queries at different grains: "
             + " vs ".join(

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/internal/materializations.py

@@ -1,4 +1,5 @@
 """Node materialization helper functions"""
+import logging
 import zlib
 from typing import Dict, List, Optional, Tuple, Union
 
@@ -40,6 +41,7 @@ from datajunction_server.sql.parsing.types import TimestampType
 from datajunction_server.utils import SEPARATOR
 
 MAX_COLUMN_NAME_LENGTH = 128
+_logger = logging.getLogger(__name__)
 
 
 async def rewrite_metrics_expressions(
@@ -193,6 +195,12 @@ async def build_non_cube_materialization_config(
     """
     Build materialization config for non-cube nodes (transforms and dimensions).
     """
+    _logger.info(
+        "Building materialization config for node=%s node_type=%s %s",
+        current_revision.name,
+        current_revision.type,
+        upsert,
+    )
     build_criteria = get_default_criteria(
         node=current_revision,
     )
@@ -254,7 +262,7 @@ async def create_new_materialization(
                 "least one cube element has a temporal partition defined",
             )
 
-        # Druid Cube (this job will take subsume all existing jobs)
+        # Druid Cube (this job will subsume all existing cube materialization types)
         if upsert.job == MaterializationJobTypeEnum.DRUID_CUBE:
             generic_config = await build_cube_materialization(
                 session=session,

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/materialization/jobs/cube_materialization.py

@@ -1,6 +1,7 @@
 """
 Cube materialization jobs
 """
+import logging
 from typing import Dict, Optional
 
 from datajunction_server.database.materialization import Materialization
@@ -26,6 +27,8 @@ from datajunction_server.service_clients import QueryServiceClient
 from datajunction_server.sql.parsing import ast
 from datajunction_server.sql.parsing.backends.antlr4 import parse
 
+_logger = logging.getLogger(__name__)
+
 
 class DefaultCubeMaterialization(
     MaterializationJob,
@@ -141,7 +144,11 @@ class DruidCubeMaterializationJob(DruidMaterializationJob, MaterializationJob):
                 "The materialization job config class must be defined!",
             )
         cube_config = self.config_class.parse_obj(materialization.config)  # type: ignore
-        return query_service_client.materialize(
+        _logger.info(
+            "Scheduling DruidCubeMaterializationJob for node=%s",
+            cube_config.cube,
+        )
+        return query_service_client.materialize_cube(
             materialization_input=DruidCubeMaterializationInput(
                 name=materialization.name,
                 cube=cube_config.cube,

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/models/cube_materialization.py

@@ -239,7 +239,7 @@ class UpsertCubeMaterialization(BaseModel):
         """
         Validates the `job` field. Converts to an enum if `job` is a string.
         """
-        if isinstance(job, str):
+        if isinstance(job, str):  # pragma: no cover
             job_name = job.upper()
             options = (
                 MaterializationJobTypeEnum._member_names_  # pylint: disable=protected-access,no-member

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/models/node_type.py

@@ -45,14 +45,3 @@ class NodeNameVersion(BaseModel):
 
     class Config:  # pylint: disable=missing-class-docstring,too-few-public-methods
         orm_mode = True
-
-    def __str__(self):
-        return f"{self.name}@{self.version}"
-
-    @classmethod
-    def from_string(cls, node_name_version: str):
-        """
-        Returns the object from a name + version string in the format node.name@v1
-        """
-        name, version = node_name_version.split("@")
-        return NodeNameVersion(name=name, version=version)

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/service_clients.py

@@ -1,4 +1,5 @@
 """Clients for various configurable services."""
+import logging
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 from urllib.parse import urljoin
@@ -31,6 +32,8 @@ from datajunction_server.sql.parsing.types import ColumnType
 if TYPE_CHECKING:
     from datajunction_server.database.engine import Engine
 
+_logger = logging.getLogger(__name__)
+
 
 class RequestsSessionWithEndpoint(requests.Session):
     """
@@ -214,13 +217,20 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         """
         Get a previously submitted query
         """
+        get_query_endpoint = f"/queries/{query_id}/"
         response = self.requests_session.get(
-            f"/queries/{query_id}/",
+            get_query_endpoint,
             headers={**self.requests_session.headers, **request_headers}
             if request_headers
             else self.requests_session.headers,
         )
         if response.status_code == 404:
+            _logger.exception(
+                "[DJQS] Failed to get query_id=%s with `GET %s`",
+                query_id,
+                get_query_endpoint,
+                exc_info=True,
+            )
             raise DJQueryServiceClientEntityNotFound(  # pragma: no cover
                 message=f"Error response from query service: {response.text}",
             )
@@ -229,6 +239,11 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
                 message=f"Error response from query service: {response.text}",
             )
         query_info = response.json()
+        _logger.info(
+            "[DJQS] Retrieved query_id=%s with `GET %s`",
+            query_id,
+            get_query_endpoint,
+        )
         return QueryWithResults(**query_info)
 
     def materialize(
@@ -236,7 +251,6 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         materialization_input: Union[
             GenericMaterializationInput,
             DruidMaterializationInput,
-            DruidCubeMaterializationInput,
         ],
         request_headers: Optional[Dict[str, str]] = None,
     ) -> MaterializationInfo:
@@ -246,7 +260,7 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         that this functionality may be moved to the materialization service at a later point.
         """
         response = self.requests_session.post(
-            "/cubes/materialize",
+            "/materialization/",
             json=materialization_input.dict(),
             headers={
                 **self.requests_session.headers,
@@ -256,10 +270,56 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
             else self.requests_session.headers,
         )
         if response.status_code not in (200, 201):  # pragma: no cover
+            _logger.exception(
+                "[DJQS] Failed to materialize node=%s with `POST /materialization/`: %s",
+                materialization_input.node_name,
+                materialization_input.dict(),
+                exc_info=True,
+            )
             return MaterializationInfo(urls=[], output_tables=[])
         result = response.json()
+        _logger.info(
+            "[DJQS] Scheduled materialization for node=%s with `POST /materialization/`",
+            materialization_input.node_name,
+        )
         return MaterializationInfo(**result)
 
+    def materialize_cube(
+        self,
+        materialization_input: DruidCubeMaterializationInput,
+        request_headers: Optional[Dict[str, str]] = None,
+    ) -> MaterializationInfo:
+        """
+        Post a request to the query service asking it to set up a scheduled materialization
+        for the node. The query service is expected to manage all reruns of this job. Note
+        that this functionality may be moved to the materialization service at a later point.
+        """
+        response = self.requests_session.post(
+            "/cubes/materialize",
+            json=materialization_input.dict(),
+            headers={
+                **self.requests_session.headers,
+                **QueryServiceClient.filtered_headers(request_headers),
+            }
+            if request_headers
+            else self.requests_session.headers,
+            timeout=20,
+        )
+        if response.status_code not in (200, 201):  # pragma: no cover
+            _logger.exception(
+                "[DJQS] Failed to schedule cube materialization for"
+                " node=%s with `POST /cubes/materialize`",
+                materialization_input.cube,
+                exc_info=True,
+            )
+            return MaterializationInfo(urls=[], output_tables=[])  # pragma: no cover
+        result = response.json()  # pragma: no cover
+        _logger.info(
+            "[DJQS] Scheduled cube materialization for node=%s with `POST /cubes/materialize`",
+            materialization_input.cube,
+        )
+        return MaterializationInfo(**result)  # pragma: no cover
+
     def deactivate_materialization(
         self,
         node_name: str,
@@ -269,8 +329,9 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         """
         Deactivates the specified node materialization
         """
+        deactivate_endpoint = f"/materialization/{node_name}/{materialization_name}/"
         response = self.requests_session.delete(
-            f"/materialization/{node_name}/{materialization_name}/",
+            deactivate_endpoint,
             headers={
                 **self.requests_session.headers,
                 **QueryServiceClient.filtered_headers(request_headers),
@@ -279,8 +340,19 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
             else self.requests_session.headers,
         )
         if response.status_code not in (200, 201):  # pragma: no cover
+            _logger.exception(
+                "[DJQS] Failed to deactivate materialization for node=%s with `DELETE %s`",
+                node_name,
+                deactivate_endpoint,
+                exc_info=True,
+            )
             return MaterializationInfo(urls=[], output_tables=[])
         result = response.json()
+        _logger.info(
+            "[DJQS] Deactivated materialization for node=%s with `DELETE %s`",
+            node_name,
+            deactivate_endpoint,
+        )
         return MaterializationInfo(**result)
 
     def get_materialization_info(
@@ -294,9 +366,12 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         """
         Gets materialization info for the node and materialization config name.
         """
-        response = self.requests_session.get(
+        info_endpoint = (
             f"/materialization/{node_name}/{node_version}/{materialization_name}/"
-            f"?node_type={node_type}",
+            f"?node_type={node_type}"
+        )
+        response = self.requests_session.get(
+            info_endpoint,
             timeout=3,
             headers={
                 **self.requests_session.headers,
@@ -306,7 +381,19 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
             else self.requests_session.headers,
         )
         if response.status_code not in (200, 201):
+            _logger.exception(
+                "[DJQS] Failed to get materialization info for node=%s with `GET %s`",
+                node_name,
+                info_endpoint,
+                exc_info=True,
+            )
             return MaterializationInfo(output_tables=[], urls=[])
+
+        _logger.info(
+            "[DJQS] Retrieved materialization info for node=%s with `GET %s`",
+            node_name,
+            info_endpoint,
+        )
         return MaterializationInfo(**response.json())
 
     def run_backfill(
@@ -319,9 +406,12 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         request_headers: Optional[Dict[str, str]] = None,
     ) -> MaterializationInfo:
         """Kicks off a backfill with the given backfill spec"""
-        response = self.requests_session.post(
+        backfill_endpoint = (
             f"/materialization/run/{node_name}/{materialization_name}"
-            f"/?node_version={node_version}&node_type={node_type}",
+            f"/?node_version={node_version}&node_type={node_type}"
+        )
+        response = self.requests_session.post(
+            backfill_endpoint,
             json=[partition.dict() for partition in partitions],
             headers={
                 **self.requests_session.headers,
@@ -332,5 +422,17 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
             timeout=20,
         )
         if response.status_code not in (200, 201):
+            _logger.exception(
+                "[DJQS] Failed to run backfill for node=%s with `POST %s`",
+                node_name,
+                backfill_endpoint,
+                exc_info=True,
+            )
             return MaterializationInfo(output_tables=[], urls=[])  # pragma: no cover
+
+        _logger.info(
+            "[DJQS] Ran backfill for node=%s with `POST %s`",
+            node_name,
+            backfill_endpoint,
+        )
         return MaterializationInfo(**response.json())

{datajunction_server-0.0.1a84.dev4 → datajunction_server-0.0.1a85.dev0}/datajunction_server/utils.py

@@ -185,7 +185,9 @@ async def refresh_if_needed(session: AsyncSession, obj, attributes: list[str]):
         await session.refresh(obj, attributes_to_refresh)
 
 
-def get_query_service_client() -> Optional[QueryServiceClient]:
+def get_query_service_client(  # pylint: disable=unused-argument
+    request: Request = None,
+) -> Optional[QueryServiceClient]:
     """
     Return query service client
     """