datajunction-server 0.0.1a84.dev2__tar.gz → 0.0.1a84.dev3__tar.gz

{datajunction_server-0.0.1a84.dev2 → datajunction_server-0.0.1a84.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a84.dev2
+Version: 0.0.1a84.dev3
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: Homepage, https://datajunction.io
 Project-URL: Repository, https://github.com/DataJunction/dj

datajunction_server-0.0.1a84.dev3/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a84.dev3"

{datajunction_server-0.0.1a84.dev2 → datajunction_server-0.0.1a84.dev3}/datajunction_server/api/cubes.py

@@ -14,6 +14,7 @@ from datajunction_server.database.node import Node
 from datajunction_server.database.user import User
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
 from datajunction_server.internal.access.authorization import validate_access
+from datajunction_server.internal.materializations import build_cube_materialization
 from datajunction_server.internal.nodes import get_cube_revision_metadata
 from datajunction_server.models import access
 from datajunction_server.models.cube import (
@@ -21,6 +22,15 @@ from datajunction_server.models.cube import (
     DimensionValue,
     DimensionValues,
 )
+from datajunction_server.models.cube_materialization import (
+    DruidCubeMaterializationInput,
+    UpsertCubeMaterialization,
+)
+from datajunction_server.models.materialization import (
+    Granularity,
+    MaterializationJobTypeEnum,
+    MaterializationStrategy,
+)
 from datajunction_server.models.metric import TranslatedSQL
 from datajunction_server.models.query import QueryCreate
 from datajunction_server.naming import from_amenable_name
@@ -47,6 +57,90 @@ async def get_cube(
     return await get_cube_revision_metadata(session, name)
 
 
+@router.get("/cubes/{name}/materialization", name="Materialization Config Cube")
+async def cube_materialization_info(
+    name: str,
+    session: AsyncSession = Depends(get_session),
+) -> DruidCubeMaterializationInput:
+    """
+    The standard cube materialization config. DJ makes sensible materialization choices
+    where possible.
+
+    Requirements:
+    - The cube must have a temporal partition column specified.
+    - The job strategy will always be "incremental time".
+
+    Outputs:
+    "measures_materializations":
+        We group the metrics by parent node. Then we try to pre-aggregate each parent node as
+        much as possible to prepare for metric queries on the cube's dimensions.
+    "combiners":
+        We combine each set of measures materializations on their shared grain. Note that we don't
+        support materializing cubes with measures materializations that don't share the same grain.
+        However, we keep `combiners` as a list in the eventual future where we support that.
+    "metrics":
+        We include a list of metrics, their required measures, and the derived expression (e.g., the
+        expression used by the metric that makes use of the pre-aggregated measures)
+
+    Once we create a scheduled materialization workflow, we freeze the metadata for that particular
+    materialized dataset. This allows us to reconstruct metrics SQL from the dataset when needed.
+    To request metrics from the materialized cube, use the metrics' measures metadata.
+    """
+    node = await Node.get_cube_by_name(session, name)
+    temporal_partitions = node.current.temporal_partition_columns()  # type: ignore
+    temporal_partition = temporal_partitions[0] if temporal_partitions else None
+    granularity_lookback_defaults = {
+        Granularity.MINUTE: "1 MINUTE",
+        Granularity.HOUR: "1 HOUR",
+        Granularity.DAY: "1 DAY",
+        Granularity.WEEK: "1 WEEK",
+        Granularity.MONTH: "1 MONTH",
+        Granularity.QUARTER: "1 QUARTER",
+        Granularity.YEAR: "1 YEAR",
+    }
+    granularity_cron_defaults = {
+        Granularity.MINUTE: "* * * * *",  # Runs every minute
+        Granularity.HOUR: "0 * * * *",  # Runs at the start of every hour
+        Granularity.DAY: "0 0 * * *",  # Runs at midnight every day
+        Granularity.WEEK: "0 0 * * 0",  # Runs at midnight on Sundays
+        Granularity.MONTH: "0 0 1 * *",  # Runs at midnight on the first of every month
+        Granularity.QUARTER: "0 0 1 */3 *",  # Runs at midnight on the first day of each quarter
+        Granularity.YEAR: "0 0 1 1 *",  # Runs at midnight on January 1st every year
+    }
+    upsert = UpsertCubeMaterialization(
+        job=MaterializationJobTypeEnum.DRUID_CUBE,
+        strategy=(
+            MaterializationStrategy.INCREMENTAL_TIME
+            if temporal_partition
+            else MaterializationStrategy.FULL
+        ),
+        lookback_window=granularity_lookback_defaults.get(
+            temporal_partition.partition.granularity,
+            granularity_lookback_defaults[Granularity.DAY],
+        ),
+        schedule=granularity_cron_defaults.get(
+            temporal_partition.partition.granularity,
+            granularity_cron_defaults[Granularity.DAY],
+        ),
+    )
+    cube_config = await build_cube_materialization(
+        session,
+        node.current,  # type: ignore
+        upsert,
+    )
+    return DruidCubeMaterializationInput(
+        name="",
+        cube=cube_config.cube,
+        dimensions=cube_config.dimensions,
+        metrics=cube_config.metrics,
+        strategy=upsert.strategy,
+        schedule=upsert.schedule,
+        job=upsert.job.name,
+        measures_materializations=cube_config.measures_materializations,
+        combiners=cube_config.combiners,
+    )
+
+
 @router.get("/cubes/{name}/dimensions/sql", name="Dimensions SQL for Cube")
 async def get_cube_dimension_sql(
     name: str,

{datajunction_server-0.0.1a84.dev2 → datajunction_server-0.0.1a84.dev3}/datajunction_server/internal/cube_materializations.py

@@ -1,5 +1,6 @@
 """Helper functions related to cube materializations."""
 import itertools
+
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from datajunction_server.construction.build_v2 import get_measures_query
@@ -56,11 +57,13 @@ def combine_measures_on_shared_grain(
         measureB.three,
         measureB.four
       FROM measureA
-      JOIN measureB ON 
+      JOIN measureB ON
         measureA.grain1 = measureB.grain1 AND
         measureA.grain2 = measureB.grain2
     """
-    measures_tables = {mat.output_table_name: mat.table_ast() for mat in measures_materializations}
+    measures_tables = {
+        mat.output_table_name: mat.table_ast() for mat in measures_materializations
+    }
     initial_mat = measures_materializations[0]
 
     # Coalesce grain fields
@@ -96,10 +99,11 @@ def combine_measures_on_shared_grain(
                     on=_combine_measures_join_criteria(
                         measures_tables[initial_mat.output_table_name],
                         measures_tables[mat.output_table_name],
-                        query_grain
+                        query_grain,
                     ),
                 ),
-            ) for mat in measures_materializations[1:]
+            )
+            for mat in measures_materializations[1:]
         ],
     )
     return ast.Query(
@@ -114,16 +118,18 @@ def _combine_measures_join_criteria(left_table, right_table, query_grain):
     """
     Generate the join condition across tables for shared grains.
     """
-    return ast.BinaryOp.And(*[
-        ast.BinaryOp.Eq(
-            ast.Column(name=ast.Name(grain), _table=left_table),
-            ast.Column(name=ast.Name(grain), _table=right_table),
-        )
-        for grain in query_grain
-    ])
+    return ast.BinaryOp.And(
+        *[
+            ast.BinaryOp.Eq(
+                ast.Column(name=ast.Name(grain), _table=left_table),
+                ast.Column(name=ast.Name(grain), _table=right_table),
+            )
+            for grain in query_grain
+        ]
+    )
 
 
-async def build_cube_materialization(
+async def build_cube_materialization(  # pylint: disable=used-before-assignment,too-many-locals
     session: AsyncSession,
     current_revision: NodeRevision,
     upsert_input: UpsertCubeMaterialization,
@@ -174,6 +180,7 @@ async def build_cube_materialization(
         measures_materialization = measures_materializations[0]
         combiners = [
             CombineMaterialization(
+                node=measures_materialization.node,
                 output_table_name=measures_materialization.output_table_name,
                 columns=measures_materialization.columns,
                 grain=measures_materialization.grain,
@@ -182,7 +189,7 @@ async def build_cube_materialization(
                 timestamp_column=measures_materialization.timestamp_column,
                 timestamp_format=measures_materialization.timestamp_format,
                 granularity=measures_materialization.granularity,
-                upstream_tables=measures_materialization.output_table_name,
+                upstream_tables=[measures_materialization.output_table_name],
             ),
         ]
     if len(measures_materializations) > 1:
@@ -252,6 +259,6 @@ async def build_cube_materialization(
         ],
         dimensions=current_revision.cube_node_dimensions,
         measures_materializations=measures_materializations,
-        combiners=combiners,
+        combiners=combiners,  # pylint: disable=possibly-used-before-assignment
     )
     return config

{datajunction_server-0.0.1a84.dev2 → datajunction_server-0.0.1a84.dev3}/datajunction_server/models/cube_materialization.py

@@ -6,13 +6,16 @@ from pydantic import BaseModel, Field, validator
 
 from datajunction_server.enum import StrEnum
 from datajunction_server.errors import DJInvalidInputException
-from datajunction_server.models.materialization import MaterializationJobTypeEnum, MaterializationStrategy, DRUID_AGG_MAPPING
-from datajunction_server.models.node_type import NodeNameVersion
-from datajunction_server.models.partition import (
-    Granularity,
+from datajunction_server.models.column import SemanticType
+from datajunction_server.models.materialization import (
+    DRUID_AGG_MAPPING,
+    MaterializationJobTypeEnum,
+    MaterializationStrategy,
 )
+from datajunction_server.models.node_type import NodeNameVersion
+from datajunction_server.models.partition import Granularity
 from datajunction_server.models.query import ColumnMetadata
-from datajunction_server.models.column import SemanticType
+
 
 class Aggregability(StrEnum):
     """
@@ -110,12 +113,9 @@ class MeasuresMaterialization(BaseModel):
         description="The time granularity for each materialization run. Examples: DAY, HOUR",
     )
 
-    spark_conf: dict[str, str] | None = Field(
+    spark_conf: Dict[str, str] | None = Field(
         description="Spark config for this materialization.",
     )
-    # druid_spec: dict[str, Any] | None = Field(
-    #     description="Druid ingestion configuration for this materialization.",
-    # )
     upstream_tables: list[str] = Field(
         description="List of upstream tables used in this materialization.",
     )
@@ -124,7 +124,9 @@ class MeasuresMaterialization(BaseModel):
         """
         Generate a unique output table name based on the parameters.
         """
-        from datajunction_server.sql.parsing import ast
+        from datajunction_server.sql.parsing import (  # pylint: disable=import-outside-toplevel
+            ast,
+        )
 
         return ast.Table(name=ast.Name(self.output_table_name))
 
@@ -154,6 +156,9 @@ class MeasuresMaterialization(BaseModel):
 
     @classmethod
     def from_measures_query(cls, measures_query, temporal_partition):
+        """
+        Builds a MeasuresMaterialization object from a measures query.
+        """
         return MeasuresMaterialization(
             node=measures_query.node,
             grain=measures_query.grain,
@@ -202,7 +207,10 @@ class CubeMetric(BaseModel):
         description="List of measures required by this metric.",
     )
     derived_expression: str = Field(
-        description="The expression for rewriting the original metric query using the materialized measures.",
+        description=(
+            "The expression for rewriting the original metric query "
+            "using the materialized measures.",
+        ),
     )
 
 
@@ -247,9 +255,12 @@ class UpsertCubeMaterialization(BaseModel):
 
 
 class CombineMaterialization(BaseModel):
+    """
+    Stage for combining measures datasets at their shared grain and ingesting to Druid.
+    Note that if there is only one upstream measures dataset, the Spark combining stage will
+    be skipped and we ingest the aggregated measures directly to Druid.
+    """
 
-    # Optional query to combine measures. If there is only one measures output
-    # then this is unnecessary.
     node: NodeNameVersion
     query: str | None
     columns: List[ColumnMetadata]
@@ -275,6 +286,9 @@ class CombineMaterialization(BaseModel):
 
     @property
     def output_table_name(self) -> str:
+        """
+        Builds an output table name based on the node and a hash of its unique key.
+        """
         unique_string = "|".join(
             [
                 self.node.name,

{datajunction_server-0.0.1a84.dev2 → datajunction_server-0.0.1a84.dev3}/datajunction_server/service_clients.py

@@ -295,7 +295,8 @@ class QueryServiceClient:  # pylint: disable=too-few-public-methods
         Gets materialization info for the node and materialization config name.
         """
         response = self.requests_session.get(
-            f"/materialization/{node_name}/{node_version}/{materialization_name}/?node_type={node_type}",
+            f"/materialization/{node_name}/{node_version}/{materialization_name}/"
+            f"?node_type={node_type}",
             timeout=3,
             headers={
                 **self.requests_session.headers,

datajunction_server-0.0.1a84.dev2/datajunction_server/__about__.py

@@ -1,4 +0,0 @@
-"""
-Version for Hatch
-"""
-__version__ = "0.0.1a84.dev2"