datajunction-server 0.0.1a52.dev0__tar.gz → 0.0.1a52.dev2__tar.gz

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-server
-Version: 0.0.1a52.dev0
+Version: 0.0.1a52.dev2
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a52.dev2/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a52.dev2"

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/datajunction_server/api/nodes.py

@@ -102,6 +102,7 @@ from datajunction_server.sql.dag import (
     _node_output_options,
     get_dimensions,
     get_downstream_nodes,
+    get_filter_only_dimensions,
     get_upstream_nodes,
 )
 from datajunction_server.sql.parsing.backends.antlr4 import parse, parse_rule
@@ -1214,7 +1215,9 @@ async def list_all_dimension_attributes(
             ],
         )
     )
-    return await get_dimensions(session, node, with_attributes=True)  # type: ignore
+    dimensions = await get_dimensions(session, node, with_attributes=True)  # type: ignore
+    filter_only_dimensions = await get_filter_only_dimensions(session, name)
+    return dimensions + filter_only_dimensions
 
 
 @router.get(

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/datajunction_server/construction/build.py

@@ -7,6 +7,7 @@ import time
 from typing import DefaultDict, Deque, Dict, List, Optional, Set, Tuple, Union, cast
 
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
 
 from datajunction_server.construction.utils import to_namespaced_name
 from datajunction_server.database import Engine
@@ -28,7 +29,7 @@ from datajunction_server.models.metric import TranslatedSQL
 from datajunction_server.models.node import BuildCriteria
 from datajunction_server.models.node_type import NodeType
 from datajunction_server.naming import LOOKUP_CHARS, amenable_name, from_amenable_name
-from datajunction_server.sql.dag import get_shared_dimensions
+from datajunction_server.sql.dag import get_dimensions, get_shared_dimensions
 from datajunction_server.sql.parsing.ast import CompileContext
 from datajunction_server.sql.parsing.backends.antlr4 import ast, parse
 from datajunction_server.sql.parsing.types import (
@@ -162,8 +163,9 @@ async def _build_joins_for_dimension_link(
     for link in join_path:
         join_query = await build_ast(session, link.join_sql_ast())
         join = join_query.select.from_.relations[-1].extensions[0]  # type: ignore
-
         # Assemble table on left of join
+        if not tables[link.node_revision]:
+            continue  # pragma: no cover
         left_table = (
             tables[link.node_revision][0].child  # type: ignore
             if isinstance(tables[link.node_revision][0], ast.Alias)
@@ -522,15 +524,15 @@ async def _build_tables_on_select(
                 node_ast,
                 copy=False,
             )
-        await select.compile(context)
-        for col in select.find_all(ast.Column):
-            if (
-                col._table
-                and isinstance(col._table, ast.Table)
-                and col._table.dj_node
-                and col._table.dj_node.name == node.name
-            ):
-                col._table = node_ast
+            await select.compile(context)
+            for col in select.find_all(ast.Column):
+                if (
+                    col._table
+                    and isinstance(col._table, ast.Table)
+                    and col._table.dj_node
+                    and col._table.dj_node.name == node.name
+                ):
+                    col._table = node_ast
 
 
 def dimension_columns_mapping(
@@ -661,6 +663,16 @@ async def add_filters_dimensions_orderby_limit_to_query_ast(
     Add filters and dimensions to a query ast
     """
     projection_addition = {}
+    node_ = await Node.get_by_name(
+        session,
+        node.name,
+        options=[
+            selectinload(Node.current).options(
+                selectinload(NodeRevision.parents).options(selectinload(Node.current)),
+            ),
+        ],
+    )
+    all_dimensions = [dim.name for dim in await get_dimensions(session, node_)]  # type: ignore
 
     if dimensions:
         for agg in dimensions:
@@ -716,8 +728,7 @@ async def add_filters_dimensions_orderby_limit_to_query_ast(
                     else:
                         col.role = col.parent.index.identifier()  # type: ignore
                     col.parent.swap(col)
-
-                if not dimensions:
+                if not dimensions and col.identifier(False) in all_dimensions:
                     projection_addition[col.identifier(False)] = col
                 if access_control:
                     dj_node = await access_control.add_request_by_node_name(
@@ -970,10 +981,40 @@ async def build_node(  # pylint: disable=too-many-arguments
             ],
         )
         access_control.validate_and_raise()
+
+    filter_asts = (
+        _deduplicate_filters(built_ast.select) if built_ast.select.where else None
+    )
+    built_ast.select.where = ast.BinaryOp.And(*filter_asts) if filter_asts else None
     _logger.info("Finished build_ast on %s", node.name)
     return built_ast
 
 
+def _deduplicate_filters(
+    select: ast.SelectExpression,
+) -> Optional[List[ast.Expression]]:
+    """
+    Takes the WHERE clause on a SELECT clause and:
+    1. Removes any duplicate filters
+    2. Removes any filters that don't compile or work with the SELECT clause
+    """
+    filters = select.where_clause_expressions_list()
+    if not filters:
+        return filters  # pragma: no cover
+
+    filter_asts: List[ast.Expression] = []
+    for filter_ in filters:
+        copied_filter = filter_.copy()
+        for reference in copied_filter.find_all(ast.Column):
+            reference.set_alias(None)
+        if all(
+            reference.table is not None
+            for reference in copied_filter.find_all(ast.Column)
+        ) and str(copied_filter) not in [str(f) for f in filter_asts]:
+            filter_asts.append(copied_filter)
+    return filter_asts
+
+
 def rename_columns(built_ast: ast.Query, node: NodeRevision):
     """
     Rename columns in the built ast to fully qualified column names.

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/datajunction_server/models/node.py

@@ -573,6 +573,7 @@ class DimensionAttributeOutput(BaseModel):
     is_primary_key: bool
     type: str
     path: List[str]
+    filter_only: bool = False
 
 
 class ColumnOutput(BaseModel):

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/datajunction_server/sql/dag.py

@@ -18,10 +18,10 @@ from datajunction_server.database.node import (
     NodeRelationship,
     NodeRevision,
 )
-from datajunction_server.errors import DJException
+from datajunction_server.errors import DJDoesNotExistException, DJException
 from datajunction_server.models.node import DimensionAttributeOutput
 from datajunction_server.models.node_type import NodeType
-from datajunction_server.utils import get_settings
+from datajunction_server.utils import SEPARATOR, get_settings
 
 settings = get_settings()
 
@@ -142,6 +142,10 @@ async def get_upstream_nodes(
         .unique()
         .scalar()
     )
+    if not node:
+        raise DJDoesNotExistException(  # pragma: no cover
+            message=f"Node with name {node_name} does not exist",
+        )
 
     dag = (
         select(
@@ -486,6 +490,44 @@ async def get_dimensions(
     return dag
 
 
+async def get_filter_only_dimensions(
+    session: AsyncSession,
+    node_name: str,
+):
+    """
+    Get dimensions for this node that can only be filtered by and cannot be grouped by
+    or retrieved as a part of the node's SELECT clause.
+    """
+    filter_only_dimensions = []
+    upstreams = await get_upstream_nodes(session, node_name, node_type=NodeType.SOURCE)
+    for upstream in upstreams:
+        await session.refresh(upstream.current, ["dimension_links"])
+        for link in upstream.current.dimension_links:
+            await session.refresh(link.dimension, ["current"])
+            await session.refresh(link.dimension.current, ["columns"])
+            column_mapping = {col.name: col for col in link.dimension.current.columns}
+            filter_only_dimensions.extend(
+                [
+                    DimensionAttributeOutput(
+                        name=dim,
+                        node_name=link.dimension.name,
+                        node_display_name=link.dimension.current.display_name,
+                        is_primary_key=(
+                            dim.split(SEPARATOR)[-1]
+                            in {
+                                col.name for col in link.dimension.current.primary_key()
+                            }
+                        ),
+                        type=str(column_mapping[dim.split(SEPARATOR)[-1]].type),
+                        path=[upstream.name],
+                        filter_only=True,
+                    )
+                    for dim in link.foreign_keys.values()
+                ],
+            )
+    return filter_only_dimensions
+
+
 async def group_dimensions_by_name(
     session: AsyncSession,
     node: Node,

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/datajunction_server/sql/parsing/ast.py

@@ -2504,6 +2504,28 @@ class SelectExpression(Aliasable, Expression):
                 projection.append(expression)
         self.projection = projection
 
+    def where_clause_expressions_list(self) -> Optional[List[Expression]]:
+        """
+        Converts the WHERE clause to a list of expressions separated by AND operators
+        """
+        if not self.where:
+            return self.where
+
+        filters = []
+        processing = collections.deque([self.where])
+        while processing:
+            current_clause = processing.pop()
+            if current_clause:
+                if (
+                    isinstance(current_clause, BinaryOp)
+                    and current_clause.op == BinaryOpKind.And
+                ):
+                    processing.append(current_clause.left)
+                    processing.append(current_clause.right)
+                else:
+                    filters.append(current_clause)
+        return filters
+
 
 class Select(SelectExpression):
     """

{datajunction_server-0.0.1a52.dev0 → datajunction_server-0.0.1a52.dev2}/tests/api/materializations_test.py

@@ -345,7 +345,6 @@ async def test_druid_measures_cube_full(
         "for node `default.repairs_cube`"
     )
     args, _ = query_service_client.materialize.call_args_list[0]  # type: ignore
-    print("args[0].query", args[0].query)
     assert str(parse(args[0].query)) == str(
         parse(load_expected_file("druid_measures_cube.full.query.sql")),
     )
@@ -571,7 +570,6 @@ WHERE repair_orders.order_date = DJ_LOGICAL_TIMESTAMP()""",
         "for node `default.repairs_cube`"
     )
     args, _ = query_service_client.materialize.call_args_list[2]  # type: ignore
-    print("args[0]", args[0].query)
     assert str(
         parse(
             args[0]