datajunction-server 0.0.1a44.dev6__tar.gz → 0.0.1a44.dev7__tar.gz

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-server
-Version: 0.0.1a44.dev6
+Version: 0.0.1a44.dev7
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a44.dev7/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a44.dev7"

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/api/data.py

@@ -103,10 +103,10 @@ def add_availability_state(
     # Merge the new availability state with the current availability state if one exists
     old_availability = node_revision.availability
     if (
-        old_availability
-        and old_availability.catalog == data.catalog
-        and old_availability.schema_ == data.schema_
-        and old_availability.table == data.table
+        node_revision.availability
+        and node_revision.availability.catalog == node.current.catalog.name
+        and node_revision.availability.schema_ == data.schema_
+        and node_revision.availability.table == data.table
     ):
         data.merge(node_revision.availability)
 

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/construction/build.py

@@ -27,12 +27,7 @@ from datajunction_server.naming import LOOKUP_CHARS, amenable_name, from_amenabl
 from datajunction_server.sql.dag import get_shared_dimensions
 from datajunction_server.sql.parsing.ast import CompileContext
 from datajunction_server.sql.parsing.backends.antlr4 import ast, parse
-from datajunction_server.sql.parsing.types import (
-    ColumnType,
-    DoubleType,
-    LongType,
-    TimestampType,
-)
+from datajunction_server.sql.parsing.types import ColumnType
 from datajunction_server.utils import SEPARATOR
 
 _logger = logging.getLogger(__name__)
@@ -736,14 +731,7 @@ def _get_node_table(
         if node.table:
             name = ast.Name(
                 node.table,
-                namespace=ast.Name(
-                    node.schema_,
-                    namespace=ast.Name(node.catalog.name)
-                    if node.schema_ == "iceberg"
-                    else None,
-                )
-                if node.schema_
-                else None,
+                namespace=ast.Name(node.schema_) if node.schema_ else None,
             )
         else:
             name = to_namespaced_name(node.name)
@@ -1438,7 +1426,6 @@ def get_measures_query(
     engine_version: Optional[str] = None,
     current_user: Optional[User] = None,
     validate_access: access.ValidateAccessFn = None,
-    cast_timestamp_to_ms: bool = False,
 ) -> TranslatedSQL:
     """
     Builds the measures SQL for a set of metrics with dimensions and filters.
@@ -1613,27 +1600,6 @@ def get_measures_query(
         for col_name, columns in dimension_grouping.items()
     ]
     combined_ast.select.projection.extend(dimension_columns)
-    for col in dimension_columns:
-        if col.type == TimestampType() and cast_timestamp_to_ms:  # type: ignore
-            col = (
-                ast.Cast(
-                    expression=ast.BinaryOp(
-                        op=ast.BinaryOpKind.Multiply,
-                        left=ast.Cast(
-                            data_type=DoubleType(),
-                            expression=col.copy().child
-                            if isinstance(col, ast.Alias)
-                            else col,
-                        ),  # type: ignore
-                        right=ast.Number(1000),
-                    ),
-                    data_type=LongType(),
-                )
-                .set_alias(ast.Name("timestamp_column"))  # type: ignore
-                .set_semantic_entity(col.semantic_entity)  # type: ignore
-                .set_semantic_type(SemanticType.TIMESTAMP)
-            )
-            combined_ast.select.projection.append(col)
 
     # Assemble column metadata
     columns_metadata = []

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/internal/materializations.py

@@ -30,7 +30,6 @@ from datajunction_server.service_clients import QueryServiceClient
 from datajunction_server.sql.parsing import ast
 from datajunction_server.sql.parsing.ast import CompileContext
 from datajunction_server.sql.parsing.backends.antlr4 import parse
-from datajunction_server.sql.parsing.types import TimestampType
 from datajunction_server.utils import SEPARATOR
 
 MAX_COLUMN_NAME_LENGTH = 128
@@ -142,7 +141,6 @@ def build_cube_materialization_config(
             dimensions=current_revision.cube_dimensions(),
             filters=[],
             validate_access=validate_access,
-            cast_timestamp_to_ms=True,
         )
         metrics_expressions = rewrite_metrics_expressions(
             session,
@@ -215,9 +213,6 @@ def create_new_materialization(
     """
     generic_config = None
     temporal_partition = current_revision.temporal_partition_columns()
-    timestamp_columns = [
-        col for col in current_revision.columns if col.type == TimestampType()
-    ]
     if current_revision.type in (
         NodeType.DIMENSION,
         NodeType.TRANSFORM,
@@ -229,7 +224,7 @@ def create_new_materialization(
         )
 
     if current_revision.type == NodeType.CUBE:
-        if not temporal_partition and not timestamp_columns:
+        if not temporal_partition:
             raise DJInvalidInputException(
                 "The cube materialization cannot be configured if there is no "
                 "temporal partition specified on the cube. Please make sure at "

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/materialization/jobs/cube_materialization.py

@@ -13,7 +13,6 @@ from datajunction_server.models.materialization import (
     DruidMeasuresCubeConfig,
     DruidMetricsCubeConfig,
     MaterializationInfo,
-    MaterializationStrategy,
 )
 from datajunction_server.naming import amenable_name
 from datajunction_server.service_clients import QueryServiceClient
@@ -71,7 +70,6 @@ class DruidMaterializationJob(MaterializationJob):
         )
         final_query = build_materialization_query(
             cube_config.query,
-            materialization,
             materialization.node_revision,
         )
         return query_service_client.materialize(
@@ -112,7 +110,6 @@ class DruidMeasuresCubeMaterializationJob(DruidMaterializationJob, Materializati
 
 def build_materialization_query(
     base_cube_query: str,
-    materialization: Materialization,
     node_revision: NodeRevision,
 ) -> ast.Query:
     """
@@ -121,29 +118,29 @@ def build_materialization_query(
     cube_materialization_query_ast = parse(
         base_cube_query.replace("${dj_logical_timestamp}", "DJ_LOGICAL_TIMESTAMP()"),
     )
+    temporal_partitions = node_revision.temporal_partition_columns()
+    temporal_partition_col = [
+        col
+        for col in cube_materialization_query_ast.select.projection
+        if col.alias_or_name.name == amenable_name(temporal_partitions[0].name)  # type: ignore
+    ]
+
     final_query = ast.Query(
         select=ast.Select(
             projection=[
                 ast.Column(name=ast.Name(col.alias_or_name.name))  # type: ignore
                 for col in cube_materialization_query_ast.select.projection
             ],
+            where=ast.BinaryOp(
+                left=ast.Column(
+                    name=ast.Name(temporal_partition_col[0].alias_or_name.name),  # type: ignore
+                ),
+                right=temporal_partitions[0].partition.temporal_expression(),
+                op=ast.BinaryOpKind.Eq,
+            ),
         ),
         ctes=cube_materialization_query_ast.ctes,
     )
-    if materialization.strategy == MaterializationStrategy.INCREMENTAL_TIME:
-        temporal_partitions = node_revision.temporal_partition_columns()
-        temporal_partition_col = [
-            col
-            for col in cube_materialization_query_ast.select.projection
-            if col.alias_or_name.name == amenable_name(temporal_partitions[0].name)  # type: ignore
-        ]
-        final_query.select.where = ast.BinaryOp(
-            left=ast.Column(
-                name=ast.Name(temporal_partition_col[0].alias_or_name.name),  # type: ignore
-            ),
-            right=temporal_partitions[0].partition.temporal_expression(),
-            op=ast.BinaryOpKind.Eq,
-        )
     combiner_cte = ast.Query(select=cube_materialization_query_ast.select).set_alias(
         ast.Name("combiner_query"),
     )

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/materialization/jobs/materialization_job.py

@@ -10,7 +10,6 @@ from datajunction_server.models.materialization import (
     GenericMaterializationConfig,
     GenericMaterializationInput,
     MaterializationInfo,
-    MaterializationStrategy,
 )
 from datajunction_server.models.partition import PartitionBackfill
 from datajunction_server.service_clients import QueryServiceClient
@@ -84,10 +83,7 @@ class SparkSqlMaterializationJob(  # pylint: disable=too-few-public-methods # pr
             ),
         )
         final_query = query_ast
-        if (
-            temporal_partitions
-            and materialization.strategy == MaterializationStrategy.INCREMENTAL_TIME
-        ):
+        if temporal_partitions:
             temporal_partition_col = [
                 col
                 for col in query_ast.select.projection

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/models/column.py

@@ -60,4 +60,3 @@ class SemanticType(StrEnum):
     MEASURE = "measure"
     METRIC = "metric"
     DIMENSION = "dimension"
-    TIMESTAMP = "timestamp"

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/models/materialization.py

@@ -6,7 +6,6 @@ from pydantic import AnyHttpUrl, BaseModel, validator
 
 from datajunction_server.enum import StrEnum
 from datajunction_server.errors import DJInvalidInputException
-from datajunction_server.models.column import SemanticType
 from datajunction_server.models.node_type import NodeType
 from datajunction_server.models.partition import (
     BackfillOutput,
@@ -15,7 +14,6 @@ from datajunction_server.models.partition import (
 )
 from datajunction_server.models.query import ColumnMetadata
 from datajunction_server.naming import amenable_name
-from datajunction_server.sql.parsing.types import TimestampType
 
 if TYPE_CHECKING:
     from datajunction_server.database.node import NodeRevision
@@ -312,35 +310,13 @@ class DruidMeasuresCubeConfig(DruidCubeConfigInput, GenericCubeConfig):
         """
         node_name = node_revision.name
         metrics_spec = list(self.metrics_spec().values())
-
-        # Either a temporal partition is configured or a timestamp column must exist on the cube
         user_defined_temporal_partitions = node_revision.temporal_partition_columns()
-        timestamp_columns = [
-            col.name for col in node_revision.columns if col.type == TimestampType()
-        ]
-        if not user_defined_temporal_partitions and not timestamp_columns:
-            raise DJInvalidInputException(  # pragma: no cover
-                "There must be at least one timestamp column or temporal partition configured"
-                " on this cube or it cannot be materialized to Druid.",
-            )
-
-        # Select which timestamp column to use: the user-defined temporal partition
-        # if it exists or a timestamp column if it doesn't
-        user_defined_temporal_partition = None
-        if user_defined_temporal_partitions:
-            user_defined_temporal_partition = user_defined_temporal_partitions[0]
-            timestamp_column = [
-                col.name
-                for col in self.columns  # type: ignore
-                if col.semantic_entity == user_defined_temporal_partition.name
-            ][0]
-        else:
-            timestamp_column = [
-                col.name
-                for col in self.columns  # type: ignore
-                if col.semantic_type == SemanticType.TIMESTAMP
-            ][0]
-
+        user_defined_temporal_partition = user_defined_temporal_partitions[0]
+        output_temporal_partition_column = [
+            col.name
+            for col in self.columns  # type: ignore
+            if col.semantic_entity == user_defined_temporal_partition.name
+        ][0]
         druid_datasource_name = (
             self.prefix  # type: ignore
             + amenable_name(node_name)  # type: ignore
@@ -355,29 +331,18 @@ class DruidMeasuresCubeConfig(DruidCubeConfigInput, GenericCubeConfig):
                     "parseSpec": {
                         "format": "parquet",
                         "dimensionsSpec": {
-                            "dimensions": sorted(
-                                # pylint: disable=no-member
-                                list(set(self.dimensions)),  # type: ignore
-                            ),
+                            "dimensions": self.dimensions,  # pylint: disable=no-member
                         },
                         "timestampSpec": {
-                            "column": timestamp_column,
-                            "format": (
-                                user_defined_temporal_partition.partition.format
-                                if user_defined_temporal_partition
-                                else "millis"
-                            ),
+                            "column": output_temporal_partition_column,
+                            "format": user_defined_temporal_partition.partition.format,
                         },
                     },
                 },
                 "metricsSpec": metrics_spec,
                 "granularitySpec": {
                     "type": "uniform",
-                    "segmentGranularity": (
-                        user_defined_temporal_partition.partition.granularity
-                        if user_defined_temporal_partition
-                        else "DAY"
-                    ),
+                    "segmentGranularity": user_defined_temporal_partition.partition.granularity,
                     "intervals": [],  # this should be set at runtime
                 },
             },

{datajunction_server-0.0.1a44.dev6 → datajunction_server-0.0.1a44.dev7}/datajunction_server/sql/parsing/ast.py

@@ -1216,7 +1216,7 @@ class TableExpression(Aliasable, Expression):
                 # For struct types we can additionally check if there's a column that matches
                 # the search column's namespace and if there's a nested field that matches the
                 # search column's name
-                if isinstance(col.type, StructType):
+                if col._type and isinstance(col.type, StructType):
                     # struct column name
                     column_namespace = ".".join(
                         [name.name for name in column.namespace],
@@ -2546,7 +2546,7 @@ class Query(TableExpression, UnNamed):
     def set_alias(self: TNode, alias: "Name") -> TNode:
         self.alias = alias
         for col in self._columns:
-            if isinstance(col, Column):
+            if isinstance(col, Column) and col.table is not None:
                 col.table.alias = self.alias
         return self