datajunction-server 0.0.1a94.dev0__tar.gz → 0.0.1a95.dev0__tar.gz

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a94.dev0
+Version: 0.0.1a95.dev0
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: Homepage, https://datajunction.io
 Project-URL: Repository, https://github.com/DataJunction/dj

datajunction_server-0.0.1a95.dev0/datajunction_server/__about__.py

@@ -0,0 +1,5 @@
+"""
+Version for Hatch
+"""
+
+__version__ = "0.0.1a95.dev0"

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/data.py

@@ -18,7 +18,7 @@ from datajunction_server.api.helpers import (
 from datajunction_server.api.sql import get_node_sql
 from datajunction_server.api.helpers import get_save_history
 from datajunction_server.database.availabilitystate import AvailabilityState
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.node import Node, NodeRevision
 from datajunction_server.database.user import User
 from datajunction_server.errors import (
@@ -31,6 +31,7 @@ from datajunction_server.internal.access.authorization import (
     validate_access_requests,
 )
 from datajunction_server.internal.engines import get_engine
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.models import access
 from datajunction_server.models.node import AvailabilityStateBase
 from datajunction_server.models.node_type import NodeType

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/graphql/resolvers/nodes.py

@@ -115,7 +115,11 @@ def load_node_revision_options(node_revision_fields):
         "cube_metrics" in node_revision_fields
         or "cube_dimensions" in node_revision_fields
     )
-    if "columns" in node_revision_fields or is_cube_request:
+    if (
+        "columns" in node_revision_fields
+        or is_cube_request
+        or "primary_key" in node_revision_fields
+    ):
         options.append(
             selectinload(DBNodeRevision.columns).options(
                 joinedload(Column.attributes).joinedload(

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/graphql/scalars/node.py

@@ -138,7 +138,14 @@ class NodeRevision:
     table: Optional[str]
 
     # Only metrics will have these fields
-    required_dimensions: Optional[List[Column]] = None
+    required_dimensions: List[Column] | None = None
+
+    @strawberry.field
+    def primary_key(self, root: "DBNodeRevision") -> list[str]:
+        """
+        The primary key of the node
+        """
+        return [col.name for col in root.primary_key()]
 
     @strawberry.field
     def metric_metadata(self, root: "DBNodeRevision") -> MetricMetadata | None:

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/helpers.py

@@ -31,7 +31,7 @@ from datajunction_server.database.attributetype import AttributeType
 from datajunction_server.database.catalog import Catalog
 from datajunction_server.database.column import Column
 from datajunction_server.database.engine import Engine
-from datajunction_server.database.history import EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.namespace import NodeNamespace
 from datajunction_server.database.node import (
     MissingParent,
@@ -49,6 +49,7 @@ from datajunction_server.errors import (
     ErrorCode,
 )
 from datajunction_server.internal.engines import get_engine
+from datajunction_server.internal.history import EntityType
 from datajunction_server.models import access
 from datajunction_server.models.attribute import RESERVED_ATTRIBUTE_NAMESPACE
 from datajunction_server.models.history import status_change_history

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/history.py

@@ -3,17 +3,26 @@ History related APIs.
 """
 
 import logging
-from typing import List
+from typing import List, Optional
 
 from fastapi import Depends, Query
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import aliased
+from sqlalchemy import and_, cast, func, String
 
 from datajunction_server.api.helpers import get_history
-from datajunction_server.database.history import EntityType, History
+from datajunction_server.database.history import History
+from datajunction_server.database.notification_preference import NotificationPreference
+from datajunction_server.database.user import User
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
+from datajunction_server.internal.history import EntityType
 from datajunction_server.models.history import HistoryOutput
-from datajunction_server.utils import get_session, get_settings
+from datajunction_server.utils import (
+    get_and_update_current_user,
+    get_session,
+    get_settings,
+)
 
 _logger = logging.getLogger(__name__)
 settings = get_settings()
@@ -44,26 +53,37 @@ async def list_history(
 
 @router.get("/history/", response_model=List[HistoryOutput])
 async def list_history_by_node_context(
-    node: str,
+    node: Optional[str] = None,
+    only_subscribed: bool = False,
     offset: int = 0,
     limit: int = Query(default=100, lte=100),
     *,
     session: AsyncSession = Depends(get_session),
+    current_user: User = Depends(get_and_update_current_user),
 ) -> List[HistoryOutput]:
     """
     List all activity history for a node context
     """
-    hist = (
-        (
-            await session.execute(
-                select(History)
-                .where(History.node == node)
-                .order_by(History.created_at.desc())
-                .offset(offset)
-                .limit(limit),
-            )
+    statement = select(History)
+    if node:
+        statement = statement.where(History.node == node)
+    if only_subscribed:
+        np_alias = aliased(NotificationPreference)
+        statement = statement.join(
+            np_alias,
+            and_(
+                cast(History.entity_type, String) == cast(np_alias.entity_type, String),
+                History.entity_name == np_alias.entity_name,
+                cast(History.activity_type, String).in_(
+                    func.array_to_string(np_alias.activity_types, ","),
+                ),
+                np_alias.user_id == current_user.id,
+            ),
         )
-        .scalars()
-        .all()
+
+    statement = (
+        statement.order_by(History.created_at.desc()).offset(offset).limit(limit)
     )
+    result = await session.execute(statement)
+    hist = result.scalars().all()
     return [HistoryOutput.from_orm(entry) for entry in hist]

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/materializations.py

@@ -16,11 +16,12 @@ from datajunction_server.api.helpers import get_save_history
 from datajunction_server.database import Node, NodeRevision
 from datajunction_server.database.backfill import Backfill
 from datajunction_server.database.column import Column, ColumnAttribute
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.user import User
 from datajunction_server.errors import DJDoesNotExistException, DJInvalidInputException
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
 from datajunction_server.internal.access.authorization import validate_access
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.internal.materializations import (
     create_new_materialization,
     schedule_materialization_jobs,

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/nodes.py

@@ -29,7 +29,7 @@ from datajunction_server.api.tags import get_tags_by_name
 from datajunction_server.database import DimensionLink
 from datajunction_server.database.attributetype import ColumnAttribute
 from datajunction_server.database.column import Column
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.node import Node, NodeRevision
 from datajunction_server.database.partition import Partition
 from datajunction_server.database.user import User
@@ -46,6 +46,7 @@ from datajunction_server.internal.access.authorization import (
     validate_access,
     validate_access_requests,
 )
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.internal.nodes import (
     activate_node,
     copy_to_new_node,

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/notifications.py

@@ -10,11 +10,12 @@ from fastapi.responses import JSONResponse
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 
-from datajunction_server.database.history import ActivityType, EntityType, History
 from datajunction_server.database.notification_preference import NotificationPreference
 from datajunction_server.database.user import User
+from datajunction_server.database.history import History
 from datajunction_server.errors import DJDoesNotExistException
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.internal.notifications import (
     get_entity_notification_preferences,
     get_user_notification_preferences,

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/tags.py

@@ -11,11 +11,12 @@ from sqlalchemy.orm import joinedload
 
 from datajunction_server.api.helpers import get_save_history
 from datajunction_server.database import Node
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.tag import Tag
 from datajunction_server.database.user import User
 from datajunction_server.errors import DJAlreadyExistsException, DJDoesNotExistException
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.models.node import NodeMinimumDetail
 from datajunction_server.models.node_type import NodeType
 from datajunction_server.models.tag import CreateTag, TagOutput, UpdateTag

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/api/users.py

@@ -10,10 +10,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload, selectinload
 
 from datajunction_server.database.column import Column
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.node import Node, NodeRevision
 from datajunction_server.database.user import User
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.models.node import NodeMinimumDetail
 from datajunction_server.models.user import UserActivity
 from datajunction_server.utils import get_session, get_settings

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/construction/build_v2.py

@@ -166,7 +166,7 @@ async def get_measures_query(
         preaggregate = preagg_requested and all(
             len(metrics2measures[metric.name][0]) > 0
             and all(
-                measure.rule.type == Aggregability.FULL
+                measure.rule.type in (Aggregability.FULL, Aggregability.LIMITED)
                 for measure in metrics2measures[metric.name][0]
             )
             for metric in children
@@ -194,13 +194,17 @@ async def get_measures_query(
             parent_ast.select.projection = [
                 expr
                 for expr in parent_ast.select.projection
-                if from_amenable_name(expr.alias_or_name.identifier(False)).split(  # type: ignore
-                    SEPARATOR,
-                )[-1]
-                in parents_to_measures[parent_node.name]
-                or from_amenable_name(expr.alias_or_name.identifier(False))  # type: ignore
-                in dimensions_without_roles
+                if (
+                    (identifier := expr.alias_or_name.identifier(False))
+                    and (
+                        from_amenable_name(identifier).split(SEPARATOR)[-1]
+                        in parents_to_measures[parent_node.name]
+                        or identifier in parents_to_measures[parent_node.name]
+                        or from_amenable_name(identifier) in dimensions_without_roles
+                    )
+                )
             ]
+
         await refresh_if_needed(session, parent_node.current, ["columns"])
         parent_ast = rename_columns(parent_ast, parent_node.current, preaggregate)
 
@@ -289,19 +293,35 @@ def build_preaggregate_query(
     built_parent_ref = parent_node.name + "_built"
     parent_node_cte = parent_ast.to_cte(ast.Name(amenable_name(built_parent_ref)))
     from_table = ast.Table(ast.Name(amenable_name(built_parent_ref)))
+
+    # Find all required GROUP BY columns based on each of the measure's aggregation rules.
+    # If the measure supports full aggregation, there are no required group-by columns, but if it
+    # supports limited aggregation, we need to aggregate to the specified level.
+    required_group_by_columns = [
+        ast.Column.from_existing(
+            parent_ast.select.column_mapping[group_by_col],
+            table=from_table,
+        )
+        for metric in metrics2measures
+        for measures in metrics2measures[metric][0]
+        if measures.rule.level
+        for group_by_col in measures.rule.level
+    ]
+
     final_query = ast.Query(
         ctes=existing_ctes + [parent_node_cte],
         select=ast.Select(
             projection=[
                 ast.Column.from_existing(col, table=from_table)
                 for col in parent_ast.select.projection
-                if col.semantic_type == SemanticType.DIMENSION  # type: ignore
+                if col and col.semantic_type == SemanticType.DIMENSION  # type: ignore
             ],
             from_=ast.From(relations=[ast.Relation(primary=from_table)]),
             group_by=[
                 ast.Column(dim.alias_or_name, _table=from_table)
                 for dim in dimensional_columns
-            ],
+            ]
+            + required_group_by_columns,
         ),
     )
 

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/database/history.py

@@ -4,50 +4,22 @@ from datetime import datetime, timezone
 from functools import partial
 from typing import Any, Dict, Optional
 
-from sqlalchemy import JSON, BigInteger, DateTime, Enum, Index, Integer, String
+from sqlalchemy import (
+    JSON,
+    BigInteger,
+    DateTime,
+    Enum,
+    Index,
+    Integer,
+    String,
+)
 from sqlalchemy.orm import Mapped, mapped_column
 
 from datajunction_server.database.base import Base
-from datajunction_server.enum import StrEnum
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.typing import UTCDatetime
 
 
-class ActivityType(StrEnum):
-    """
-    An activity type
-    """
-
-    CREATE = "create"
-    DELETE = "delete"
-    RESTORE = "restore"
-    UPDATE = "update"
-    REFRESH = "refresh"
-    TAG = "tag"
-    SET_ATTRIBUTE = "set_attribute"
-    STATUS_CHANGE = "status_change"
-
-
-class EntityType(StrEnum):
-    """
-    An entity type for which activity can occur
-    """
-
-    ATTRIBUTE = "attribute"
-    AVAILABILITY = "availability"
-    BACKFILL = "backfill"
-    CATALOG = "catalog"
-    COLUMN_ATTRIBUTE = "column_attribute"
-    DEPENDENCY = "dependency"
-    ENGINE = "engine"
-    LINK = "link"
-    MATERIALIZATION = "materialization"
-    NAMESPACE = "namespace"
-    NODE = "node"
-    PARTITION = "partition"
-    QUERY = "query"
-    TAG = "tag"
-
-
 class History(Base):
     """
     An event to store as part of the server's activity history

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/database/notification_preference.py

@@ -16,7 +16,7 @@ from sqlalchemy import (
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from datajunction_server.database.base import Base
-from datajunction_server.database.history import ActivityType, EntityType
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.database.user import User
 from datajunction_server.typing import UTCDatetime
 

datajunction_server-0.0.1a95.dev0/datajunction_server/internal/history.py

@@ -0,0 +1,39 @@
+"""Enums for history events"""
+
+from datajunction_server.enum import StrEnum
+
+
+class ActivityType(StrEnum):
+    """
+    An activity type
+    """
+
+    CREATE = "create"
+    DELETE = "delete"
+    RESTORE = "restore"
+    UPDATE = "update"
+    REFRESH = "refresh"
+    TAG = "tag"
+    SET_ATTRIBUTE = "set_attribute"
+    STATUS_CHANGE = "status_change"
+
+
+class EntityType(StrEnum):
+    """
+    An entity type for which activity can occur
+    """
+
+    ATTRIBUTE = "attribute"
+    AVAILABILITY = "availability"
+    BACKFILL = "backfill"
+    CATALOG = "catalog"
+    COLUMN_ATTRIBUTE = "column_attribute"
+    DEPENDENCY = "dependency"
+    ENGINE = "engine"
+    LINK = "link"
+    MATERIALIZATION = "materialization"
+    NAMESPACE = "namespace"
+    NODE = "node"
+    PARTITION = "partition"
+    QUERY = "query"
+    TAG = "tag"

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/internal/namespaces.py

@@ -12,7 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import joinedload
 
 from datajunction_server.api.helpers import get_node_namespace
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.namespace import NodeNamespace
 from datajunction_server.database.node import Column, Node, NodeRevision
 from datajunction_server.database.user import User
@@ -21,6 +21,7 @@ from datajunction_server.errors import (
     DJDoesNotExistException,
     DJInvalidInputException,
 )
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.internal.nodes import (
     get_cube_revision_metadata,
     hard_delete_node,

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/internal/nodes.py

@@ -24,7 +24,7 @@ from datajunction_server.construction.build_v2 import compile_node_ast
 from datajunction_server.database.attributetype import AttributeType, ColumnAttribute
 from datajunction_server.database.column import Column
 from datajunction_server.database.dimensionlink import DimensionLink
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
 from datajunction_server.database.materialization import Materialization
 from datajunction_server.database.metricmetadata import MetricMetadata
 from datajunction_server.database.node import MissingParent, Node, NodeRevision
@@ -42,6 +42,7 @@ from datajunction_server.internal.materializations import (
     create_new_materialization,
     schedule_materialization_jobs,
 )
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.internal.validation import NodeValidator, validate_node_data
 from datajunction_server.models import access
 from datajunction_server.models.attribute import (

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/internal/notifications.py

@@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import selectinload
 
 from datajunction_server.database.notification_preference import NotificationPreference
-from datajunction_server.database.history import EntityType
+from datajunction_server.internal.history import EntityType
 
 
 async def get_user_notification_preferences(

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/models/history.py

@@ -6,7 +6,8 @@ from typing import TYPE_CHECKING, Any, Dict, Optional
 
 from pydantic.main import BaseModel
 
-from datajunction_server.database.history import ActivityType, EntityType, History
+from datajunction_server.database.history import History
+from datajunction_server.internal.history import ActivityType, EntityType
 from datajunction_server.typing import UTCDatetime
 
 if TYPE_CHECKING:

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/models/notifications.py

@@ -4,7 +4,7 @@ from typing import List, Optional
 
 from pydantic.main import BaseModel
 
-from datajunction_server.database.history import ActivityType, EntityType
+from datajunction_server.internal.history import ActivityType, EntityType
 
 
 class NotificationPreferenceModel(BaseModel):

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/sql/decompose.py

@@ -104,6 +104,11 @@ class MeasureExtractor:
                     type=Aggregability.FULL
                     if func.quantifier != ast.SetQuantifier.Distinct
                     else Aggregability.LIMITED,
+                    level=(
+                        [str(arg) for arg in func.args]
+                        if func.quantifier == ast.SetQuantifier.Distinct
+                        else None
+                    ),
                 ),
             ),
         ]
@@ -126,6 +131,11 @@ class MeasureExtractor:
                     type=Aggregability.FULL
                     if func.quantifier != ast.SetQuantifier.Distinct
                     else Aggregability.LIMITED,
+                    level=(
+                        [str(arg) for arg in func.args]
+                        if func.quantifier == ast.SetQuantifier.Distinct
+                        else None
+                    ),
                 ),
             ),
             Measure(
@@ -136,6 +146,11 @@ class MeasureExtractor:
                     type=Aggregability.FULL
                     if func.quantifier != ast.SetQuantifier.Distinct
                     else Aggregability.LIMITED,
+                    level=(
+                        [str(arg) for arg in func.args]
+                        if func.quantifier == ast.SetQuantifier.Distinct
+                        else None
+                    ),
                 ),
             ),
         ]

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/datajunction_server/sql/parsing/ast.py

@@ -846,7 +846,11 @@ class Column(Aliasable, Named, Expression):
         return column_namespace, column_name, subscript_name
 
     @classmethod
-    def from_existing(cls, col: Aliasable | Expression, table: "TableExpression"):
+    def from_existing(
+        cls,
+        col: Union[Aliasable, Expression, "Column"],
+        table: "TableExpression",
+    ):
         """
         Build a selectable column from an existing one
         """

{datajunction_server-0.0.1a94.dev0 → datajunction_server-0.0.1a95.dev0}/tests/api/graphql/find_nodes_test.py

@@ -592,6 +592,7 @@ async def test_find_transform(
                         name
                     }
                 }
+                primaryKey
             }
         }
     }
@@ -617,6 +618,7 @@ async def test_find_transform(
                 ],
                 "extractedMeasures": None,
                 "metricMetadata": None,
+                "primaryKey": [],
             },
             "name": "default.repair_orders_fact",
             "type": "TRANSFORM",