datajunction-server 0.0.1a98.dev1__tar.gz → 0.0.1a99.dev0__tar.gz

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a98.dev1
+Version: 0.0.1a99.dev0
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: Homepage, https://datajunction.io
 Project-URL: Repository, https://github.com/DataJunction/dj

datajunction_server-0.0.1a99.dev0/datajunction_server/__about__.py

@@ -0,0 +1,5 @@
+"""
+Version for Hatch
+"""
+
+__version__ = "0.0.1a99.dev0"

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/api/helpers.py

@@ -201,6 +201,8 @@ async def get_query(
     engine: Optional[Engine] = None,
     access_control: Optional[access.AccessControlStore] = None,
     use_materialized: bool = True,
+    query_parameters: Optional[Dict[str, str]] = None,
+    ignore_errors: bool = True,
 ) -> ast.Query:
     """
     Get a query for a metric, dimensions, and filters
@@ -214,12 +216,14 @@ async def get_query(
         node.current,  # type: ignore
         use_materialized=use_materialized,
     )
-    query_ast = await (
+    if ignore_errors:
         query_builder.ignore_errors()
-        .with_access_control(access_control)
+    query_ast = await (
+        query_builder.with_access_control(access_control)
         .with_build_criteria(build_criteria)
         .add_dimensions(dimensions)
         .add_filters(filters)
+        .add_query_parameters(query_parameters)
         .limit(limit)
         .order_by(orderby)
         .build()
@@ -605,6 +609,7 @@ async def build_sql_for_multiple_metrics(
     access_control: Optional[access.AccessControlStore] = None,
     ignore_errors: bool = True,
     use_materialized: bool = True,
+    query_parameters: Optional[Dict[str, str]] = None,
 ) -> Tuple[TranslatedSQL, Engine, Catalog]:
     """
     Build SQL for multiple metrics. Used by both /sql and /data endpoints
@@ -730,6 +735,7 @@ async def build_sql_for_multiple_metrics(
         limit=limit,
         access_control=access_control,
         ignore_errors=ignore_errors,
+        query_parameters=query_parameters,
     )
     columns = [
         assemble_column_metadata(col)  # type: ignore

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/api/nodes.py

@@ -106,8 +106,8 @@ from datajunction_server.service_clients import QueryServiceClient
 from datajunction_server.sql.dag import (
     _node_output_options,
     get_dimensions,
+    get_dimensions_graph,
     get_downstream_nodes,
-    get_filter_only_dimensions,
     get_upstream_nodes,
 )
 from datajunction_server.sql.parsing.backends.antlr4 import parse, parse_rule
@@ -1561,27 +1561,24 @@ async def list_all_dimension_attributes(
     *,
     depth: int = 30,
     session: AsyncSession = Depends(get_session),
-) -> List[DimensionAttributeOutput]:
+) -> list[DimensionAttributeOutput]:
     """
     List all available dimension attributes for the given node.
     """
-    node = await Node.get_by_name(
-        session,
-        name,
-        options=[
-            joinedload(Node.current).options(
-                joinedload(NodeRevision.parents).options(joinedload(Node.current)),
-            ),
-        ],
-    )
-    dimensions = await get_dimensions(
-        session,
-        node,  # type: ignore
-        with_attributes=True,
-        depth=depth,
-    )
-    filter_only_dimensions = await get_filter_only_dimensions(session, name)
-    return dimensions + filter_only_dimensions
+    dims = await get_dimensions_graph(session, name)
+    dimensions_map = {dim.id: dim for dim, _ in dims}
+    return [
+        DimensionAttributeOutput(
+            name=f"{dim.name}.{col.name}",
+            node_name=dim.name,
+            node_display_name=dim.current.display_name,
+            properties=[],  # attribute_types.split(",") if attribute_types else [],
+            type=str(col.type),
+            path=[dimensions_map[int(node_id)].name for node_id in path],
+        )
+        for dim, path in dims
+        for col in dim.current.columns
+    ]
 
 
 @router.get(

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/api/sql.py

@@ -2,10 +2,11 @@
 SQL related APIs.
 """
 
+import json
 import logging
 from collections import OrderedDict
 from http import HTTPStatus
-from typing import List, Optional, Tuple, cast
+from typing import Any, List, Optional, Tuple, cast
 
 from fastapi import BackgroundTasks, Depends, Query
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -58,6 +59,7 @@ async def get_measures_sql_for_cube_v2(
             "subsequent queries are more efficient."
         ),
     ),
+    query_params: str = Query("{}", description="Query parameters"),
     *,
     include_all_columns: bool = Query(
         False,
@@ -106,6 +108,7 @@ async def get_measures_sql_for_cube_v2(
         sql_transpilation_library=settings.sql_transpilation_library,
         use_materialized=use_materialized,
         preagg_requested=preaggregate,
+        query_parameters=json.loads(query_params),
     )
     return measures_query
 
@@ -122,6 +125,7 @@ async def build_and_save_node_sql(
     access_control: AccessControlStore,
     ignore_errors: bool = True,
     use_materialized: bool = True,
+    query_parameters: dict[str, Any] | None = None,
 ) -> QueryRequest:
     """
     Build node SQL and save it to query requests
@@ -152,6 +156,7 @@ async def build_and_save_node_sql(
             engine_version=engine.version if engine else None,
             access_control=access_control,
             use_materialized=use_materialized,
+            query_parameters=query_parameters,
         )
         # We save the request for both the cube and the metrics, so that if someone makes either
         # of these types of requests, they'll go to the cached query
@@ -160,6 +165,8 @@ async def build_and_save_node_sql(
             ([node_name], QueryBuildType.NODE),
         ]
         for nodes, query_type in requests_to_save:
+            if query_parameters:
+                continue  # pragma: no cover
             request = await QueryRequest.save_query_request(
                 session=session,
                 nodes=nodes,
@@ -190,6 +197,7 @@ async def build_and_save_node_sql(
             access_control=access_control,
             ignore_errors=ignore_errors,
             use_materialized=use_materialized,
+            query_parameters=query_parameters,
         )
         query = translated_sql.sql
         columns = translated_sql.columns
@@ -204,6 +212,8 @@ async def build_and_save_node_sql(
             engine=engine,
             access_control=access_control,
             use_materialized=use_materialized,
+            query_parameters=query_parameters,
+            ignore_errors=ignore_errors,
         )
         columns = [
             assemble_column_metadata(col)  # type: ignore
@@ -242,6 +252,7 @@ async def get_node_sql(
     background_tasks: BackgroundTasks,
     ignore_errors: bool = True,
     use_materialized: bool = True,
+    query_parameters: dict[str, Any] | None = None,
 ) -> Tuple[TranslatedSQL, QueryRequest]:
     """
     Return SQL for a node.
@@ -283,6 +294,7 @@ async def get_node_sql(
             engine=engine,
             access_control=access_control,
             use_materialized=use_materialized,
+            query_parameters=query_parameters,
         )
         return (
             TranslatedSQL(
@@ -304,6 +316,7 @@ async def get_node_sql(
         access_control=access_control,
         ignore_errors=ignore_errors,
         use_materialized=use_materialized,
+        query_parameters=query_parameters,
     )
     return (
         TranslatedSQL(
@@ -326,6 +339,7 @@ async def get_sql(
     filters: List[str] = Query([]),
     orderby: List[str] = Query([]),
     limit: Optional[int] = None,
+    query_params: str = Query("{}", description="Query parameters"),
     *,
     session: AsyncSession = Depends(get_session),
     engine_name: Optional[str] = None,
@@ -355,6 +369,7 @@ async def get_sql(
         background_tasks=background_tasks,
         ignore_errors=ignore_errors,  # type: ignore
         use_materialized=use_materialized,  # type: ignore
+        query_parameters=json.loads(query_params),
     )
     return translated_sql
 
@@ -366,6 +381,7 @@ async def get_sql_for_metrics(
     filters: List[str] = Query([]),
     orderby: List[str] = Query([]),
     limit: Optional[int] = None,
+    query_params: str = Query("{}", description="Query parameters"),
     *,
     session: AsyncSession = Depends(get_session),
     engine_name: Optional[str] = None,
@@ -402,19 +418,21 @@ async def get_sql_for_metrics(
             http_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
         )
 
-    if query_request := await QueryRequest.get_query_request(
-        session,
-        nodes=metrics,
-        dimensions=dimensions,
-        filters=filters,
-        orderby=orderby,
-        limit=limit,
-        engine_name=engine_name,
-        engine_version=engine_version,
-        query_type=QueryBuildType.METRICS,
-    ):
+    if (
+        query_request := await QueryRequest.get_query_request(
+            session,
+            nodes=metrics,
+            dimensions=dimensions,
+            filters=filters,
+            orderby=orderby,
+            limit=limit,
+            engine_name=engine_name,
+            engine_version=engine_version,
+            query_type=QueryBuildType.METRICS,
+        )
+    ) and not query_params:
         # Update the node SQL in a background task to keep it up-to-date
-        background_tasks.add_task(
+        background_tasks.add_task(  # pragma: no cover
             build_and_save_sql_for_metrics,
             session=session,
             metrics=metrics,
@@ -427,13 +445,14 @@ async def get_sql_for_metrics(
             access_control=access_control,
             ignore_errors=ignore_errors,
             use_materialized=use_materialized,
+            query_parameters=json.loads(query_params),
         )
-        engine = (
+        engine = (  # pragma: no cover
             await get_engine(session, engine_name, engine_version)  # type: ignore
             if engine_name
             else None
         )
-        return TranslatedSQL(
+        return TranslatedSQL(  # pragma: no cover
             sql=query_request.query,
             columns=query_request.columns,
             dialect=engine.dialect if engine else None,
@@ -451,6 +470,7 @@ async def get_sql_for_metrics(
         access_control,
         ignore_errors=ignore_errors,  # type: ignore
         use_materialized=use_materialized,  # type: ignore
+        query_parameters=json.loads(query_params),
     )
 
 
@@ -466,6 +486,7 @@ async def build_and_save_sql_for_metrics(
     access_control: Optional[access.AccessControlStore] = None,
     ignore_errors: bool = True,
     use_materialized: bool = True,
+    query_parameters: dict[str, Any] | None = None,
 ):
     """
     Builds and saves SQL for metrics.
@@ -482,6 +503,7 @@ async def build_and_save_sql_for_metrics(
         access_control,
         ignore_errors=ignore_errors,  # type: ignore
         use_materialized=use_materialized,  # type: ignore
+        query_parameters=query_parameters,
     )
 
     await QueryRequest.save_query_request(

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/construction/build.py

@@ -4,7 +4,7 @@ import collections
 import logging
 import os
 from concurrent.futures import ThreadPoolExecutor
-from typing import DefaultDict, List, Optional, Set, Tuple
+from typing import Any, DefaultDict, List, Optional, Set, Tuple
 
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -164,6 +164,7 @@ async def build_metric_nodes(
     build_criteria: Optional[BuildCriteria] = None,
     access_control: Optional[access.AccessControlStore] = None,
     ignore_errors: bool = True,
+    query_parameters: Optional[dict[str, Any]] = None,
 ):
     """
     Build a single query for all metrics in the list, including the specified
@@ -195,6 +196,7 @@ async def build_metric_nodes(
     builder = (
         builder.add_filters(filters)
         .add_dimensions(dimensions)
+        .add_query_parameters(query_parameters)
         .order_by(orderby)
         .limit(limit)
         .with_build_criteria(build_criteria)

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/construction/build_v2.py

@@ -104,6 +104,7 @@ async def get_measures_query(
     sql_transpilation_library: Optional[str] = None,
     use_materialized: bool = True,
     preagg_requested: bool = False,
+    query_parameters: dict[str, Any] = None,
 ) -> List[GeneratedSQL]:
     """
     Builds the measures SQL for a set of metrics with dimensions and filters.
@@ -185,6 +186,7 @@ async def get_measures_query(
             .with_build_criteria(build_criteria)
             .add_dimensions(dimensions)
             .add_filters(filters)
+            .add_query_parameters(query_parameters)
             .order_by(orderby)
             .build()
         )
@@ -380,6 +382,7 @@ class QueryBuilder:
         self.use_materialized = use_materialized
 
         self._filters: List[str] = []
+        self._parameters: dict[str, ast.Value] = {}
         self._required_dimensions: List[str] = [
             required.name for required in self.node_revision.required_dimensions
         ]
@@ -439,6 +442,18 @@ class QueryBuilder:
             self.filter_by(filter_)
         return self
 
+    def add_query_parameters(
+        self,
+        query_parameters: dict[str, ast.Value | Any] | None = None,
+    ):
+        """Add parameters to the query builder."""
+        for param, value in (query_parameters or {}).items():
+            self._parameters[param] = QueryBuilder.normalize_query_param_value(
+                param,
+                value,
+            )
+        return self
+
     def add_dimension(self, dimension: str):
         """Add dimension to the query builder."""
         if (
@@ -499,6 +514,13 @@ class QueryBuilder:
         """All filters"""
         return self._filters
 
+    @property
+    def parameters(self) -> dict[str, ast.Value]:
+        """
+        Extracts parameters from relevant filters
+        """
+        return self._parameters
+
     @property
     def filter_asts(self) -> List[ast.Expression]:
         """
@@ -593,6 +615,18 @@ class QueryBuilder:
                     order=order,
                 )
 
+        # Replace any parameters in the final AST with their values
+        for param in self.final_ast.find_all(ast.QueryParameter):  # type: ignore
+            if param.name in self.parameters and param.parent:
+                param.parent.replace(param, self.parameters[param.name])
+            else:
+                self.errors.append(
+                    DJQueryBuildError(
+                        code=ErrorCode.MISSING_PARAMETER,
+                        message=f"Missing value for parameter: {param.name}",
+                    ),
+                )
+
         # Error validation
         self.validate_access()
         if self.errors and not self._ignore_errors:
@@ -860,6 +894,24 @@ class QueryBuilder:
                     dimension_node_joins[dim_node].requested_dimensions.append(dim)
         return dimension_node_joins
 
+    @classmethod
+    def normalize_query_param_value(cls, param: str, value: ast.Value | Any):
+        match value:
+            case ast.Value():
+                return value
+            case bool():
+                return ast.Boolean(value)
+            case int() | float():
+                return ast.Number(value)
+            case None:
+                return ast.Null()
+            case str():
+                return ast.String(f"'{value}'")
+            case _:
+                raise TypeError(
+                    f"Unsupported parameter type: {type(value)} for param {param}",
+                )
+
 
 class CubeQueryBuilder:
     """
@@ -888,6 +940,7 @@ class CubeQueryBuilder:
         self._dimensions: List[str] = []
         self._orderby: List[str] = []
         self._limit: Optional[int] = None
+        self._parameters: dict[str, ast.Value] = {}
         self._build_criteria: Optional[BuildCriteria] = self.get_default_criteria()
         self._access_control: Optional[access.AccessControlStore] = None
         self._ignore_errors: bool = False
@@ -968,6 +1021,15 @@ class CubeQueryBuilder:
             self.add_dimension(dimension)
         return self
 
+    def add_query_parameters(self, query_parameters: dict[str, Any] | None = None):
+        """Add parameters to the query builder."""
+        for param, value in (query_parameters or {}).items():
+            self._parameters[param] = QueryBuilder.normalize_query_param_value(
+                param,
+                value,
+            )
+        return self
+
     def order_by(self, orderby: Optional[Union[str, List[str]]] = None):
         """Set order by for the query builder."""
         if isinstance(orderby, str):
@@ -1013,6 +1075,13 @@ class CubeQueryBuilder:
         """All filters"""
         return self._filters
 
+    @property
+    def parameters(self) -> dict[str, ast.Value]:
+        """
+        Extracts parameters from relevant filters
+        """
+        return self._parameters
+
     async def build(self) -> ast.Query:
         """
         Builds SQL for multiple metrics with the requested set of dimensions,
@@ -1119,6 +1188,7 @@ class CubeQueryBuilder:
                 .with_build_criteria(self._build_criteria)
                 .add_dimensions(self.dimensions)
                 .add_filters(self.filters)
+                .add_query_parameters(self.parameters)
                 .build()
             )
             self.errors.extend(query_builder.errors)

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/errors.py

@@ -40,6 +40,7 @@ class ErrorCode(IntEnum):
     COMPOUND_BUILD_EXCEPTION = 300
     MISSING_PARENT = 301
     TYPE_INFERENCE = 302
+    MISSING_PARAMETER = 303
 
     # Authentication
     AUTHENTICATION_ERROR = 400

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/models/node.py

@@ -580,11 +580,11 @@ class DimensionAttributeOutput(BaseModel):
     """
 
     name: str
-    node_name: Optional[str]
-    node_display_name: Optional[str]
+    node_name: str | None
+    node_display_name: str | None
     properties: list[str] | None
-    type: str
-    path: List[str]
+    type: str | None
+    path: list[str]
     filter_only: bool = False
 
 

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/sql/dag.py

@@ -9,6 +9,7 @@ from sqlalchemy import and_, func, join, literal, or_, select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import aliased, contains_eager, joinedload, selectinload
 from sqlalchemy.sql.operators import is_
+from sqlalchemy.dialects.postgresql import array
 
 from datajunction_server.database.attributetype import AttributeType, ColumnAttribute
 from datajunction_server.database.column import Column
@@ -226,6 +227,95 @@ async def get_upstream_nodes(
     ]
 
 
+async def get_dimensions_graph(
+    session: AsyncSession,
+    node_name: str,
+    node_type: NodeType = None,
+    include_deactivated: bool = True,
+) -> list[tuple[Node, list[str]]]:
+    """
+    Gets all upstreams of the given node, filterable by node type.
+    Uses a recursive CTE query to build out all parents of the node.
+    """
+    node = (
+        (
+            await session.execute(
+                select(Node)
+                .where(
+                    (Node.name == node_name) & (is_(Node.deactivated_at, None)),
+                )
+                .options(joinedload(Node.current)),
+            )
+        )
+        .unique()
+        .scalar()
+    )
+    if not node:
+        raise DJDoesNotExistException(  # pragma: no cover
+            message=f"Node with name {node_name} does not exist",
+        )
+
+    dag = (
+        (
+            select(
+                DimensionLink.node_revision_id,
+                NodeRevision.id,
+                NodeRevision.node_id,
+                array([NodeRevision.node_id]).label("join_path"),  # start path
+            )
+            .where(DimensionLink.node_revision_id == node.current.id)
+            .join(Node, DimensionLink.dimension_id == Node.id)
+            .join(
+                NodeRevision,
+                (Node.id == NodeRevision.node_id)
+                & (Node.current_version == NodeRevision.version),
+            )
+        )
+        .cte("dimensions", recursive=True)
+        .suffix_with(
+            "CYCLE node_id SET is_cycle USING path",
+        )
+    )
+
+    paths = dag.union_all(
+        select(
+            dag.c.node_revision_id,
+            NodeRevision.id,
+            NodeRevision.node_id,
+            func.array_cat(dag.c.join_path, array([NodeRevision.node_id])).label(
+                "join_path",
+            ),
+        )
+        .join(DimensionLink, dag.c.id == DimensionLink.node_revision_id)
+        .join(Node, DimensionLink.dimension_id == Node.id)
+        .join(
+            NodeRevision,
+            (Node.id == NodeRevision.node_id)
+            & (Node.current_version == NodeRevision.version),
+        ),
+    )
+
+    node_selector = select(Node, paths.c.join_path)
+    if not include_deactivated:
+        node_selector = node_selector.where(is_(Node.deactivated_at, None))
+    statement = (
+        node_selector.join(paths, paths.c.node_id == Node.id)
+        .join(
+            NodeRevision,
+            (Node.current_version == NodeRevision.version)
+            & (Node.id == NodeRevision.node_id),
+        )
+        .options(*_node_output_options())
+    )
+    results = await session.execute(statement)
+    rows = results.all()
+    return [
+        (node, path)
+        for node, path in rows
+        if node.type == node_type or node_type is None
+    ]
+
+
 async def get_dimensions_dag(
     session: AsyncSession,
     node_revision: NodeRevision,

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/sql/parsing/ast.py

@@ -9,6 +9,7 @@ from dataclasses import dataclass, field, fields
 from enum import Enum
 from functools import reduce
 from itertools import chain, zip_longest
+import re
 from typing import (
     Any,
     Callable,
@@ -2603,6 +2604,35 @@ class SelectExpression(Aliasable, Expression):
         return {col.alias_or_name.name: col for col in self.projection}
 
 
+@dataclass(eq=False)
+class QueryParameter(Expression):
+    """
+    Represents a query parameter that can be substituted in a query
+    """
+
+    name: str
+    prefix: str = ":"
+    quote_style: str = ""
+    _type: Optional["ColumnType"] = field(repr=False, default=None)
+
+    def __str__(self) -> str:
+        return self.identifier(quotes=True)
+
+    def identifier(self, quotes: bool = True) -> str:
+        """
+        The full parameter name, including the prefix and quotes
+        """
+        quote_style = "" if not quotes else self.quote_style
+        return f"{self.prefix}{quote_style}{self.name}{quote_style}"
+
+    @property
+    def type(self) -> ColumnType:
+        """
+        The type of the parameter
+        """
+        return self._type if self._type else StringType()
+
+
 class Select(SelectExpression):
     """
     A single select statement type

{datajunction_server-0.0.1a98.dev1 → datajunction_server-0.0.1a99.dev0}/datajunction_server/sql/parsing/backends/antlr4.py

@@ -3,6 +3,7 @@ import copy
 import inspect
 import logging
 from functools import lru_cache
+import re
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast
 
 import antlr4
@@ -656,6 +657,16 @@ def _(ctx: sbp.NumericLiteralContext):
     return ast.Number(ctx.number().getText())
 
 
+@visit.register
+def _(ctx: sbp.ParameterLiteralContext):
+    raw = ctx.getText()
+    match = re.match(r'^:("?`?)([\w.]+)(`?"?)$', raw)
+    if not match:
+        raise ValueError(f"Invalid query parameter format: {raw}")
+    param_name = match.group(2)
+    return ast.QueryParameter(name=param_name, quote_style=match.group(3))
+
+
 @visit.register
 def _(ctx: sbp.StringLitContext):
     if string := ctx.STRING():