datajunction-server 0.0.1a46.dev1__tar.gz → 0.0.1a46.dev3__tar.gz

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-server
-Version: 0.0.1a46.dev1
+Version: 0.0.1a46.dev3
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a46.dev3/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a46.dev3"

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/api/data.py

@@ -12,12 +12,10 @@ from sqlalchemy.orm import joinedload, selectinload
 from sse_starlette.sse import EventSourceResponse
 
 from datajunction_server.api.helpers import (
-    assemble_column_metadata,
     build_sql_for_multiple_metrics,
-    get_query,
     query_event_stream,
-    validate_orderby,
 )
+from datajunction_server.api.sql import get_sql
 from datajunction_server.database.availabilitystate import AvailabilityState
 from datajunction_server.database.history import ActivityType, EntityType, History
 from datajunction_server.database.node import Node, NodeRevision
@@ -34,7 +32,6 @@ from datajunction_server.internal.access.authorization import (
 )
 from datajunction_server.internal.engines import get_engine
 from datajunction_server.models import access
-from datajunction_server.models.metric import TranslatedSQL
 from datajunction_server.models.node import AvailabilityStateBase
 from datajunction_server.models.node_type import NodeType
 from datajunction_server.models.query import QueryCreate, QueryWithResults
@@ -188,6 +185,18 @@ async def get_data(  # pylint: disable=too-many-locals
     """
     Gets data for a node
     """
+    query = await get_sql(
+        node_name,
+        dimensions,
+        filters,
+        orderby,
+        limit,
+        session=session,
+        engine_name=engine_name,
+        engine_version=engine_version,
+        current_user=current_user,
+        validate_access=validate_access,
+    )
     node = await Node.get_by_name(session, node_name, raise_if_not_exists=True)
     available_engines = node.current.catalog.engines  # type: ignore
     engine = (
@@ -200,34 +209,6 @@ async def get_data(  # pylint: disable=too-many-locals
             f"The selected engine is not available for the node {node_name}. "
             f"Available engines include: {', '.join(engine.name for engine in available_engines)}",
         )
-    validate_orderby(orderby, [node_name], dimensions)
-
-    access_control = access.AccessControlStore(
-        validate_access=validate_access,
-        user=current_user,
-        base_verb=access.ResourceRequestVerb.EXECUTE,
-    )
-
-    query_ast = await get_query(
-        session=session,
-        node_name=node_name,
-        dimensions=dimensions,
-        filters=filters,
-        orderby=orderby,
-        limit=limit,
-        engine=engine,
-        access_control=access_control,
-    )
-
-    columns = [
-        assemble_column_metadata(col)  # type: ignore
-        for col in query_ast.select.projection
-    ]
-
-    query = TranslatedSQL(
-        sql=str(query_ast),
-        columns=columns,
-    )
 
     query_create = QueryCreate(
         engine_name=engine.name,
@@ -242,7 +223,7 @@ async def get_data(  # pylint: disable=too-many-locals
     )
     # Inject column info if there are results
     if result.results.__root__:  # pragma: no cover
-        result.results.__root__[0].columns = columns
+        result.results.__root__[0].columns = query.columns
     return result
 
 

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/api/helpers.py

@@ -359,7 +359,7 @@ async def validate_cube(  # pylint: disable=too-many-locals
     session: AsyncSession,
     metric_names: List[str],
     dimension_names: List[str],
-    require_dimensions: bool = True,
+    require_dimensions: bool = False,
 ) -> Tuple[List[Column], List[Node], List[Node], List[Column], Optional[Catalog]]:
     """
     Validate that a set of metrics and dimensions can be built together.

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/api/sql.py

@@ -131,6 +131,7 @@ async def get_sql(  # pylint: disable=too-many-locals
     """
     Return SQL for a node.
     """
+    dimensions = [dim for dim in dimensions if dim and dim != ""]
     access_control = access.AccessControlStore(
         validate_access=validate_access,
         user=UserOutput.from_orm(current_user) if current_user else None,

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/construction/build.py

@@ -463,11 +463,16 @@ async def _build_tables_on_select(
                                 if (  # pragma: no cover
                                     col.alias_or_name.name in foreign_keys_alias_map
                                 ):
-                                    col.name = foreign_keys_alias_map[  # type: ignore
+                                    key = foreign_keys_alias_map[  # type: ignore
                                         col.alias_or_name.name
-                                    ].name
+                                    ]
+                                    col.name = (
+                                        key.name
+                                        if isinstance(key, ast.Named)
+                                        else key.alias_or_name  # type: ignore
+                                    )
                             filter_asts.append(
-                                temp_select.where,  # type:ignore
+                                temp_select.where,  # type: ignore
                             )
                     if filter_asts:
                         node_query.select.where = ast.BinaryOp.And(*filter_asts)

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/database/queryrequest.py

@@ -319,7 +319,7 @@ class QueryRequest(Base):  # type: ignore  # pylint: disable=too-few-public-meth
         invalid_dimensions = sorted(
             list(set(dimensions).difference(available_dimensions)),
         )
-        if invalid_dimensions:
+        if dimensions and invalid_dimensions:
             raise DJInvalidInputException(
                 f"{', '.join(invalid_dimensions)} are not available "
                 f"dimensions on {', '.join(nodes)}",

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/internal/nodes.py

@@ -305,6 +305,7 @@ async def create_cube_node_revision(  # pylint: disable=too-many-locals
         session,
         data.metrics,
         data.dimensions,
+        require_dimensions=True,
     )
     status = (
         NodeStatus.VALID
@@ -1843,8 +1844,8 @@ async def revalidate_node(  # pylint: disable=too-many-locals,too-many-statement
                 existing_col.type = col.type
                 updated_columns = True
         else:
-            node.current.columns.append(col)  # type: ignore
-            updated_columns = True
+            node.current.columns.append(col)  # type: ignore  # pragma: no cover
+            updated_columns = True  # pragma: no cover
 
     # Only create a new revision if the columns have been updated
     if updated_columns:  # type: ignore

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/datajunction_server/sql/functions.py

@@ -1395,9 +1395,11 @@ def infer_type(arg: Union[ct.StringType, ct.TimestampType]) -> ct.ColumnType:
 
 class DateAdd(Function):
     """
-    Adds a specified number of days to a date.
+    date_add(date|timestamp|str, int) - Adds a specified number of days to a date.
     """
 
+    dialects = [Dialect.SPARK]
+
 
 @DateAdd.register  # type: ignore
 def infer_type(
@@ -1407,6 +1409,14 @@ def infer_type(
     return ct.DateType()
 
 
+@DateAdd.register  # type: ignore
+def infer_type(
+    start_date: ct.TimestampType,
+    days: ct.IntegerBase,
+) -> ct.DateType:
+    return ct.DateType()
+
+
 @DateAdd.register  # type: ignore
 def infer_type(
     start_date: ct.StringType,
@@ -3686,6 +3696,24 @@ def infer_type(_: ct.ColumnType) -> ct.IntegerType:
     return ct.IntegerType()
 
 
+class RegexpExtract(Function):
+    """
+    regexp_extract(str, regexp[, idx]) - Extract the first string in the str that
+    match the regexp expression and corresponding to the regex group index.
+    """
+
+    dialects = [Dialect.SPARK, Dialect.DRUID]
+
+
+@RegexpExtract.register
+def infer_type(  # type: ignore
+    str_: ct.StringType,
+    regexp: ct.StringType,
+    idx: Optional[ct.IntegerType] = 1,
+) -> ct.StringType:
+    return ct.StringType()
+
+
 class RegexpLike(Function):
     """
     regexp_like(str, regexp) - Returns true if str matches regexp, or false otherwise
@@ -3702,6 +3730,25 @@ def infer_type(  # type: ignore
     return ct.BooleanType()
 
 
+class RegexpReplace(Function):
+    """
+    regexp_replace(str, regexp, rep[, position]) - Replaces all substrings of str that
+    match regexp with rep.
+    """
+
+    dialects = [Dialect.SPARK, Dialect.DRUID]
+
+
+@RegexpReplace.register
+def infer_type(  # type: ignore
+    str_: ct.StringType,
+    regexp: ct.StringType,
+    rep: ct.StringType,
+    position: Optional[ct.IntegerType] = 1,
+) -> ct.StringType:
+    return ct.StringType()
+
+
 class Replace(Function):
     """
     replace(str, search[, replace]) - Replaces all occurrences of `search` with `replace`.

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/tests/api/data_test.py

@@ -40,8 +40,11 @@ class TestDataForNode:
             },
         )
         data = response.json()
-        assert response.status_code == 500
-        assert "Cannot resolve type of column something" in data["message"]
+        assert response.status_code == 422
+        assert (
+            "something are not available dimensions on default.payment_type"
+            in data["message"]
+        )
 
     @pytest.mark.asyncio
     async def test_get_dimension_data(
@@ -467,7 +470,7 @@ class TestDataForNode:
         data = response.json()
         assert data["message"] == (
             "Authorization of User `dj` for this request failed."
-            "\nThe following requests were denied:\nexecute:node/basic.num_comments."
+            "\nThe following requests were denied:\nread:node/basic.num_comments."
         )
         assert response.status_code == 403
         app.dependency_overrides.clear()

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/tests/api/sql_test.py

@@ -1516,6 +1516,45 @@ async def test_saving_metrics_sql_requests(  # pylint: disable=too-many-statemen
                 ],
             ],
         ),
+        # metric without any dimensions or filters should return single aggregate value
+        (
+            ["ROADS"],
+            "default.num_repair_orders",
+            [],
+            [],
+            """
+                SELECT  count(default_DOT_repair_orders_fact.repair_order_id) default_DOT_num_repair_orders
+                 FROM (SELECT  default_DOT_repair_orders.repair_order_id,
+                    default_DOT_repair_orders.municipality_id,
+                    default_DOT_repair_orders.hard_hat_id,
+                    default_DOT_repair_orders.dispatcher_id,
+                    default_DOT_repair_orders.order_date,
+                    default_DOT_repair_orders.dispatched_date,
+                    default_DOT_repair_orders.required_date,
+                    default_DOT_repair_order_details.discount,
+                    default_DOT_repair_order_details.price,
+                    default_DOT_repair_order_details.quantity,
+                    default_DOT_repair_order_details.repair_type_id,
+                    default_DOT_repair_order_details.price * default_DOT_repair_order_details.quantity AS total_repair_cost,
+                    default_DOT_repair_orders.dispatched_date - default_DOT_repair_orders.order_date AS time_to_dispatch,
+                    default_DOT_repair_orders.dispatched_date - default_DOT_repair_orders.required_date AS dispatch_delay
+                 FROM roads.repair_orders AS default_DOT_repair_orders JOIN roads.repair_order_details AS default_DOT_repair_order_details ON default_DOT_repair_orders.repair_order_id = default_DOT_repair_order_details.repair_order_id)
+                 AS default_DOT_repair_orders_fact
+                """,
+            [
+                {
+                    "column": "default_DOT_num_repair_orders",
+                    "name": "default_DOT_num_repair_orders",
+                    "node": "default.num_repair_orders",
+                    "type": "bigint",
+                    "semantic_type": None,
+                    "semantic_entity": "default.num_repair_orders.default_DOT_num_repair_orders",
+                },
+            ],
+            [
+                [25],
+            ],
+        ),
     ],
 )
 @pytest.mark.asyncio

{datajunction_server-0.0.1a46.dev1 → datajunction_server-0.0.1a46.dev3}/tests/sql/functions_test.py

@@ -3103,6 +3103,27 @@ async def test_rank(session: AsyncSession):
     assert query.select.projection[3].type == ct.IntegerType()  # type: ignore
 
 
+@pytest.mark.asyncio
+async def test_regexp_extract(session: AsyncSession):
+    """
+    Test `regexp_extract`
+    """
+    # w/o position arg
+    query = parse("SELECT regexp_extract('100-200', '(\\d+)')")
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.StringType()  # type: ignore
+    # w/ position arg
+    query = parse("SELECT regexp_extract('100-200', '(\\d+)', 42)")
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.StringType()  # type: ignore
+
+
 @pytest.mark.asyncio
 async def test_regexp_like(session: AsyncSession):
     """
@@ -3118,6 +3139,61 @@ async def test_regexp_like(session: AsyncSession):
     assert query.select.projection[0].type == ct.BooleanType()  # type: ignore
 
 
+@pytest.mark.asyncio
+async def test_regexp_replace(session: AsyncSession):
+    """
+    Test `regexp_replace`
+    """
+    # w/o position arg
+    query = parse("SELECT regexp_replace('100-200', '(\\d+)', 'num')")
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.StringType()  # type: ignore
+    # w/ position arg
+    query = parse("SELECT regexp_replace('100-200', '(\\d+)', 'num', 42)")
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.StringType()  # type: ignore
+
+
+@pytest.mark.asyncio
+async def test_date_add(session: AsyncSession):
+    """
+    Test `date_add`
+    """
+    # first arg: date
+    query = parse(
+        "SELECT date_add(CURRENT_DATE(), 42)",
+    )
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.DateType()  # type: ignore
+    # first arg: timestamp
+    query = parse(
+        "SELECT date_add(CURRENT_TIMESTAMP(), 42)",
+    )
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.DateType()  # type: ignore
+    # first arg: string
+    query = parse(
+        "SELECT date_add('2020-01-01', 42)",
+    )
+    exc = DJException()
+    ctx = ast.CompileContext(session=session, exception=exc)
+    await query.compile(ctx)
+    assert not exc.errors
+    assert query.select.projection[0].type == ct.DateType()  # type: ignore
+
+
 @pytest.mark.asyncio
 async def test_replace(session: AsyncSession):
     """

datajunction_server-0.0.1a46.dev1/datajunction_server/__about__.py

@@ -1,4 +0,0 @@
-"""
-Version for Hatch
-"""
-__version__ = "0.0.1a46.dev1"