datajunction-server 0.0.1a67__tar.gz → 0.0.1a68.dev0__tar.gz

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a67
+Version: 0.0.1a68.dev0
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a68.dev0/alembic/versions/2024_10_05_1707-f953dc7ec5a5_add_indexes_on_history.py

@@ -0,0 +1,28 @@
+"""Add indexes on history
+
+Revision ID: f953dc7ec5a5
+Revises: f3c9b40deb6f
+Create Date: 2024-10-05 17:07:17.320467+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'f953dc7ec5a5'
+down_revision = 'f3c9b40deb6f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('history', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_history_entity_name'), ['entity_name'], unique=False)
+
+
+def downgrade():
+    with op.batch_alter_table('history', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_history_entity_name'))

datajunction_server-0.0.1a68.dev0/alembic/versions/2024_10_05_1713-6b30d917f125_add_indexes_on_history2.py

@@ -0,0 +1,28 @@
+"""Add indexes on history2
+
+Revision ID: 6b30d917f125
+Revises: f953dc7ec5a5
+Create Date: 2024-10-05 17:13:00.623878+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '6b30d917f125'
+down_revision = 'f953dc7ec5a5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('history', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_history_user'), ['user'], unique=False)
+
+
+def downgrade():
+    with op.batch_alter_table('history', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_history_user'))

datajunction_server-0.0.1a68.dev0/alembic/versions/2024_10_05_1720-90d6250dc394_add_indexes_on_nodes.py

@@ -0,0 +1,29 @@
+"""Add indexes on nodes
+
+Revision ID: 90d6250dc394
+Revises: 6b30d917f125
+Create Date: 2024-10-05 17:20:13.317534+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '90d6250dc394'
+down_revision = '6b30d917f125'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('node', schema=None) as batch_op:
+        batch_op.create_index('cursor_index', ['created_at', 'id'], unique=False, postgresql_using='btree')
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('node', schema=None) as batch_op:
+        batch_op.drop_index('cursor_index', postgresql_using='btree')

datajunction_server-0.0.1a68.dev0/alembic/versions/2024_10_06_0447-1c13a84b41d6_add_indexes_on_node_namespace.py

@@ -0,0 +1,28 @@
+"""Add indexes on node namespace
+
+Revision ID: 1c13a84b41d6
+Revises: 90d6250dc394
+Create Date: 2024-10-06 04:47:18.214335+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '1c13a84b41d6'
+down_revision = '90d6250dc394'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('node', schema=None) as batch_op:
+        batch_op.create_index('namespace_index', ['namespace'], unique=False, postgresql_using='btree', postgresql_ops={"identifier": "varchar_pattern_ops"},)
+
+
+def downgrade():
+    with op.batch_alter_table('node', schema=None) as batch_op:
+        batch_op.drop_index('namespace_index', postgresql_using='text_pattern_ops')

datajunction_server-0.0.1a68.dev0/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a68.dev0"

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/api/helpers.py

@@ -739,7 +739,7 @@ async def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,
                 # It must be either a Source node or some type of Materialized node.
                 # Even if there is no availability state for it right now, this qery may be built
                 # for future execution, so we don't need to check for availability state here.
-                if tbl.dj_node
+                if tbl.dj_node and tbl.dj_node.type == NodeType.SOURCE
             ],
         ),
         engine,

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/api/nodes.py

@@ -67,9 +67,9 @@ from datajunction_server.internal.validation import validate_node_data
 from datajunction_server.models import access
 from datajunction_server.models.attribute import AttributeTypeIdentifier
 from datajunction_server.models.dimensionlink import (
+    JoinLinkInput,
     JoinType,
     LinkDimensionIdentifier,
-    LinkDimensionInput,
 )
 from datajunction_server.models.node import (
     ColumnOutput,
@@ -874,7 +874,7 @@ async def link_dimension(
                 " These column types are incompatible and the dimension cannot be linked",
             )
 
-    link_input = LinkDimensionInput(
+    link_input = JoinLinkInput(
         dimension_node=dimension,
         join_type=JoinType.LEFT,
         join_on=(
@@ -910,10 +910,130 @@ async def link_dimension(
     )
 
 
+@router.post("/nodes/{node_name}/columns/{node_column}/link", status_code=201)
+async def add_reference_dimension_link(
+    node_name: str,
+    node_column: str,
+    dimension_node: str,
+    dimension_column: str,
+    role: Optional[str] = None,
+    session: AsyncSession = Depends(get_session),
+    current_user: User = Depends(get_and_update_current_user),
+) -> JSONResponse:
+    """
+    Add reference dimension link to a node column
+    """
+    node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)
+    dim_node = await Node.get_by_name(session, dimension_node, raise_if_not_exists=True)
+    if dim_node.type != NodeType.DIMENSION:  # type: ignore
+        raise DJInvalidInputException(
+            message=f"Node {node.name} is not of type dimension!",  # type: ignore
+        )
+
+    # The target and dimension columns should both exist
+    target_column = await get_column(session, node.current, node_column)  # type: ignore
+    dim_column = await get_column(session, dim_node.current, dimension_column)  # type: ignore
+
+    # Check the dimension column's type is compatible with the target column's type
+    if not dim_column.type.is_compatible(target_column.type):
+        raise DJInvalidInputException(
+            f"The column {target_column.name} has type {target_column.type} "
+            f"and is being linked to the dimension {dimension_node} "
+            f"via the dimension column {dimension_column}, which has "
+            f"type {dim_column.type}. These column types are incompatible"
+            " and the dimension cannot be linked",
+        )
+
+    activity_type = (
+        ActivityType.UPDATE if target_column.dimension_column else ActivityType.CREATE
+    )
+
+    # Create the reference link
+    target_column.dimension_id = dim_node.id  # type: ignore
+    target_column.dimension_column = (
+        f"{dimension_column}[{role}]" if role else dimension_column
+    )
+    session.add(target_column)
+    session.add(
+        History(
+            entity_type=EntityType.LINK,
+            entity_name=node.name,  # type: ignore
+            node=node.name,  # type: ignore
+            activity_type=activity_type,
+            details={
+                "node_name": node_name,  # type: ignore
+                "node_column": node_column,
+                "dimension_node": dimension_node,
+                "dimension_column": dimension_column,
+                "role": role,
+            },
+            user=current_user.username,
+        ),
+    )
+    await session.commit()
+    return JSONResponse(
+        status_code=201,
+        content={
+            "message": (
+                f"{node_name}.{node_column} has been successfully "
+                f"linked to {dimension_node}.{dimension_column}"
+            ),
+        },
+    )
+
+
+@router.delete("/nodes/{node_name}/columns/{node_column}/link", status_code=201)
+async def remove_reference_dimension_link(
+    node_name: str,
+    node_column: str,
+    session: AsyncSession = Depends(get_session),
+    current_user: User = Depends(get_and_update_current_user),
+) -> JSONResponse:
+    """
+    Remove reference dimension link from a node column
+    """
+    node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)
+    target_column = await get_column(session, node.current, node_column)  # type: ignore
+    if target_column.dimension_id or target_column.dimension_column:
+        target_column.dimension_id = None
+        target_column.dimension_column = None
+        session.add(target_column)
+        session.add(
+            History(
+                entity_type=EntityType.LINK,
+                entity_name=node.name,  # type: ignore
+                node=node.name,  # type: ignore
+                activity_type=ActivityType.DELETE,
+                details={
+                    "node_name": node_name,  # type: ignore
+                    "node_column": node_column,
+                },
+                user=current_user.username,
+            ),
+        )
+        session.commit()
+        return JSONResponse(
+            status_code=200,
+            content={
+                "message": (
+                    f"The reference dimension link on {node_name}.{node_column} has been removed."
+                ),
+            },
+        )
+    return JSONResponse(
+        status_code=200,
+        content={
+            "message": (
+                f"There is no reference dimension link on {node_name}.{node_column}."
+            ),
+        },
+    )
+
+
 @router.post("/nodes/{node_name}/link/", status_code=201)
 async def add_complex_dimension_link(  # pylint: disable=too-many-locals
     node_name: str,
-    link_input: LinkDimensionInput,
+    link_input: JoinLinkInput,
     session: AsyncSession = Depends(get_session),
     current_user: User = Depends(get_and_update_current_user),
 ) -> JSONResponse:
@@ -1337,7 +1457,7 @@ async def list_node_dag(
     name="List All Dimension Attributes",
 )
 async def list_all_dimension_attributes(
-    name: str, *, session: AsyncSession = Depends(get_session)
+    name: str, *, depth: int = 30, session: AsyncSession = Depends(get_session)
 ) -> List[DimensionAttributeOutput]:
     """
     List all available dimension attributes for the given node.
@@ -1353,7 +1473,12 @@ async def list_all_dimension_attributes(
             ],
         )
     )
-    dimensions = await get_dimensions(session, node, with_attributes=True)  # type: ignore
+    dimensions = await get_dimensions(
+        session,
+        node,  # type: ignore
+        with_attributes=True,
+        depth=depth,
+    )
     filter_only_dimensions = await get_filter_only_dimensions(session, name)
     return dimensions + filter_only_dimensions
 

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/construction/build_v2.py

@@ -693,7 +693,7 @@ class QueryBuilder:  # pylint: disable=too-many-instance-attributes,too-many-pub
                     self.node_revision,
                     dimension_attr.name,
                 )
-                if not join_path:
+                if not join_path and join_path != []:
                     self.errors.append(
                         DJQueryBuildError(
                             code=ErrorCode.INVALID_DIMENSION_JOIN,
@@ -733,6 +733,15 @@ def get_column_from_canonical_dimension(
     if dimension_attr.node_name == node.name:
         column_name = dimension_attr.column_name
 
+    # Dimension requested has reference link on node
+    for column in node.columns:
+        if (
+            column.dimension
+            and column.dimension.name == dimension_attr.node_name
+            and column.dimension_column == dimension_attr.column_name
+        ):
+            column_name = column.name
+
     # Dimension referenced was foreign key of dimension link
     link = next(
         (
@@ -781,14 +790,18 @@ async def dimension_join_path(
     list of dimension links that represent the join path
     """
     # Check if it is a local dimension
-    if dimension.startswith(node.name):
-        for col in node.columns:  # pragma: no cover
-            # Decide if we should restrict this to only columns marked as dimensional
-            # await session.refresh(col, ["attributes"]) TODO
-            # if col.is_dimensional():
-            #     ...
-            if f"{node.name}.{col.name}" == dimension:
-                return []
+    for col in node.columns:  # pragma: no cover
+        # Decide if we should restrict this to only columns marked as dimensional
+        # await session.refresh(col, ["attributes"]) TODO
+        # if col.is_dimensional():
+        #     ...
+        if f"{node.name}.{col.name}" == dimension:
+            return []
+        if (
+            col.dimension
+            and f"{col.dimension.name}.{col.dimension_column}" == dimension
+        ):
+            return []
 
     dimension_attr = FullColumnName(dimension)
 
@@ -801,7 +814,7 @@ async def dimension_join_path(
         [(link, [link]) for link in node.dimension_links],
     )
     while processing_queue:
-        current_link, join_path = processing_queue.pop()
+        current_link, join_path = processing_queue.popleft()
         await session.refresh(current_link, ["dimension"])
         if current_link.dimension.name == dimension_attr.node_name:
             return join_path

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/internal/nodes.py

@@ -50,8 +50,8 @@ from datajunction_server.models.attribute import (
 from datajunction_server.models.base import labelize
 from datajunction_server.models.cube import CubeElementMetadata, CubeRevisionMetadata
 from datajunction_server.models.dimensionlink import (
+    JoinLinkInput,
     LinkDimensionIdentifier,
-    LinkDimensionInput,
 )
 from datajunction_server.models.history import status_change_history
 from datajunction_server.models.materialization import (
@@ -1486,7 +1486,7 @@ async def get_cube_revision_metadata(session: AsyncSession, name: str):
 async def upsert_complex_dimension_link(
     session: AsyncSession,
     node_name: str,
-    link_input: LinkDimensionInput,
+    link_input: JoinLinkInput,
     current_user: User,
 ) -> ActivityType:
     """

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/models/dimensionlink.py

@@ -39,9 +39,9 @@ class LinkDimensionIdentifier(BaseModel):
     role: Optional[str]
 
 
-class LinkDimensionInput(BaseModel):
+class JoinLinkInput(BaseModel):
     """
-    Input for linking a dimension to a node
+    Input for creating a join link between a dimension node and node
     """
 
     dimension_node: str

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/sql/dag.py

@@ -86,7 +86,9 @@ async def get_downstream_nodes(
     )
     if not include_cubes:
         initial_dag = initial_dag.where((NodeRevision.type != NodeType.CUBE))
-    dag = initial_dag.cte("downstreams", recursive=True)
+    dag = initial_dag.cte("downstreams", recursive=True).suffix_with(
+        "CYCLE node_id SET is_cycle USING path",
+    )
 
     next_layer = (
         select(
@@ -166,19 +168,25 @@ async def get_upstream_nodes(
         )
 
     dag = (
-        select(
-            NodeRelationship.child_id,
-            NodeRevision.id,
-            NodeRevision.node_id,
+        (
+            select(
+                NodeRelationship.child_id,
+                NodeRevision.id,
+                NodeRevision.node_id,
+            )
+            .where(NodeRelationship.child_id == node.current.id)
+            .join(Node, NodeRelationship.parent_id == Node.id)
+            .join(
+                NodeRevision,
+                (Node.id == NodeRevision.node_id)
+                & (Node.current_version == NodeRevision.version),
+            )
         )
-        .where(NodeRelationship.child_id == node.current.id)
-        .join(Node, NodeRelationship.parent_id == Node.id)
-        .join(
-            NodeRevision,
-            (Node.id == NodeRevision.node_id)
-            & (Node.current_version == NodeRevision.version),
+        .cte("upstreams", recursive=True)
+        .suffix_with(
+            "CYCLE node_id SET is_cycle USING path",
         )
-    ).cte("upstreams", recursive=True)
+    )
 
     paths = dag.union_all(
         select(
@@ -220,6 +228,7 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
     session: AsyncSession,
     node_revision: NodeRevision,
     with_attributes: bool = True,
+    depth: int = 30,
 ) -> List[Union[DimensionAttributeOutput, Node]]:
     """
     Gets the dimensions graph of the given node revision with a single recursive CTE query.
@@ -245,6 +254,7 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
                 NodeColumns.node_id.label("node_revision_id"),
                 Column.dimension_id,
                 Column.name,
+                Column.dimension_column,
             )
             .select_from(NodeColumns)
             .join(Column, NodeColumns.column_id == Column.id)
@@ -261,6 +271,7 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
                     )
                     + literal("]")
                 ).label("name"),
+                literal(None).label("dimension_column"),
             ).select_from(DimensionLink),
         )
         .cte("graph_branches")
@@ -271,6 +282,7 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
         select(
             initial_node.id.label("path_start"),
             graph_branches.c.name.label("col_name"),
+            graph_branches.c.dimension_column.label("dimension_column"),
             dimension_node.id.label("path_end"),
             (
                 initial_node.name
@@ -284,6 +296,7 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
             dimension_node.name.label("node_name"),
             dimension_rev.id.label("node_revision_id"),
             dimension_rev.display_name.label("node_display_name"),
+            literal(0).label("depth"),
         )
         .select_from(initial_node)
         .join(graph_branches, node_revision.id == graph_branches.c.node_revision_id)
@@ -301,11 +314,15 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
         )
         .where(initial_node.id == node_revision.id)
     ).cte("dimensions_graph", recursive=True)
+    dimensions_graph = dimensions_graph.suffix_with(
+        "CYCLE node_revision_id SET is_cycle USING path",
+    )
 
     paths = dimensions_graph.union_all(
         select(
             dimensions_graph.c.path_start,
             graph_branches.c.name.label("col_name"),
+            graph_branches.c.dimension_column.label("dimension_column"),
             next_node.id.label("path_end"),
             (
                 dimensions_graph.c.join_path
@@ -319,10 +336,12 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
             next_node.name.label("node_name"),
             next_rev.id.label("node_revision_id"),
             next_rev.display_name.label("node_display_name"),
-        ).select_from(
+            (dimensions_graph.c.depth + literal(1)).label("depth"),
+        )
+        .select_from(
             dimensions_graph.join(
                 current_node,
-                dimensions_graph.c.path_end == current_node.id,
+                (dimensions_graph.c.path_end == current_node.id),
             )
             .join(
                 current_rev,
@@ -331,10 +350,15 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
                     current_rev.node_id == current_node.id,
                 ),
             )
-            .join(graph_branches, current_rev.id == graph_branches.c.node_revision_id)
+            .join(
+                graph_branches,
+                (current_rev.id == graph_branches.c.node_revision_id)
+                & (is_(graph_branches.c.dimension_column, None)),
+            )
             .join(
                 next_node,
                 (next_node.id == graph_branches.c.dimension_id)
+                & (is_(graph_branches.c.dimension_column, None))
                 & (is_(next_node.deactivated_at, None)),
             )
             .join(
@@ -344,7 +368,8 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
                     next_rev.node_id == next_node.id,
                 ),
             ),
-        ),
+        )
+        .where(dimensions_graph.c.depth <= depth),
     )
 
     # Final SELECT statements
@@ -380,7 +405,16 @@ async def get_dimensions_dag(  # pylint: disable=too-many-locals
         )
         .select_from(paths)
         .join(NodeColumns, NodeColumns.node_id == paths.c.node_revision_id)
-        .join(column, NodeColumns.column_id == column.id)
+        .join(
+            column,
+            and_(
+                NodeColumns.column_id == column.id,
+                or_(
+                    is_(paths.c.dimension_column, None),
+                    paths.c.dimension_column == column.name,
+                ),
+            ),
+        )
         .join(ColumnAttribute, column.id == ColumnAttribute.column_id, isouter=True)
         .join(
             AttributeType,
@@ -490,6 +524,7 @@ async def get_dimensions(
     session: AsyncSession,
     node: Node,
     with_attributes: bool = True,
+    depth: int = 30,
 ) -> List[Union[DimensionAttributeOutput, Node]]:
     """
     Return all available dimensions for a given node.
@@ -501,10 +536,16 @@ async def get_dimensions(
             session,
             node.current.parents[0].current,
             with_attributes,
+            depth=depth,
         )
     else:
         await session.refresh(node, attribute_names=["current"])
-        dag = await get_dimensions_dag(session, node.current, with_attributes)
+        dag = await get_dimensions_dag(
+            session,
+            node.current,
+            with_attributes,
+            depth=depth,
+        )
     return dag
 
 

{datajunction_server-0.0.1a67 → datajunction_server-0.0.1a68.dev0}/datajunction_server/sql/parsing/ast.py

@@ -2682,7 +2682,11 @@ class Query(TableExpression, UnNamed):
             for cte in (nearest_query.ctes if nearest_query else [])
         }
         table_options = (
-            list(self.select.from_.find_all(TableExpression))
+            [
+                tbl
+                for tbl in self.select.from_.find_all(TableExpression)
+                if tbl.get_nearest_parent_of_type(Query) is self
+            ]
             if self.select.from_
             else []
         )