datajunction-server 0.0.1a47.dev0__tar.gz → 0.0.1a48__tar.gz

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/Makefile

@@ -18,7 +18,7 @@ docker-run:
 	docker compose up
 
 test:
-	pdm run pytest -n auto --cov=datajunction_server --cov-report=html -vv tests/ --doctest-modules datajunction_server --without-integration --without-slow-integration ${PYTEST_ARGS}
+	pdm run pytest -n auto --cov=datajunction_server -vv tests/ --doctest-modules datajunction_server --without-integration --without-slow-integration ${PYTEST_ARGS}
 
 integration:
 	pdm run pytest --cov=dj -vv tests/ --doctest-modules datajunction_server --with-integration --with-slow-integration

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.1a47.dev0
+Version: 0.0.1a48
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a48/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a48"

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/api/nodes.py

@@ -622,7 +622,6 @@ async def create_cube(
     """
     Create a cube node.
     """
-    print("data", data)
     await raise_if_node_exists(session, data.name)
 
     # if the node previously existed and now is inactive

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/construction/build.py

@@ -773,6 +773,9 @@ def _get_node_table(
     If a node has a materialization available, return the materialized table
     """
     table = None
+    can_use_materialization = (
+        build_criteria and node.name != build_criteria.target_node_name
+    )
     if node.type == NodeType.SOURCE:
         if node.table:
             name = ast.Name(
@@ -796,8 +799,12 @@ def _get_node_table(
             ],
             _dj_node=node,
         )
-    elif node.availability and node.availability.is_available(
-        criteria=build_criteria,
+    elif (
+        can_use_materialization
+        and node.availability
+        and node.availability.is_available(
+            criteria=build_criteria,
+        )
     ):  # pragma: no cover
         table = ast.Table(
             ast.Name(
@@ -822,6 +829,23 @@ def _get_node_table(
     return table
 
 
+def get_default_criteria(
+    node: NodeRevision,
+) -> BuildCriteria:
+    """
+    Get the default build criteria for a node.
+    """
+    return BuildCriteria(
+        # set the dialect by finding available engines for this node, or default to Spark
+        dialect=(
+            node.catalog.engines[0].dialect
+            if node.catalog and node.catalog.engines and node.catalog.engines[0].dialect
+            else Dialect.SPARK
+        ),
+        target_node_name=node.name,
+    )
+
+
 async def build_node(  # pylint: disable=too-many-arguments
     session: AsyncSession,
     node: NodeRevision,
@@ -844,15 +868,7 @@ async def build_node(  # pylint: disable=too-many-arguments
 
     # Set the dialect by finding available engines for this node, or default to Spark
     if not build_criteria:
-        build_criteria = BuildCriteria(
-            dialect=(
-                node.catalog.engines[0].dialect
-                if node.catalog
-                and node.catalog.engines
-                and node.catalog.engines[0].dialect
-                else Dialect.SPARK
-            ),
-        )
+        build_criteria = get_default_criteria(node)
 
     # get dimension columns which are required
     # in the stated bound dimensions on the metric node

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/database/availabilitystate.py

@@ -70,7 +70,8 @@ class AvailabilityState(Base):  # pylint: disable=too-few-public-methods
         """
         Determine whether an availability state is useable given criteria
         """
-        # Criteria to determine if an availability state should be used needs to be added
+        # TODO: we should evaluate this availability state against the criteria. # pylint: disable=fixme
+        #       Remember that VTTS can be also evaluated at runtime dependency.
         return True
 
 

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/internal/materializations.py

@@ -7,7 +7,11 @@ from sqlalchemy.exc import InvalidRequestError
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from datajunction_server.api.helpers import build_sql_for_multiple_metrics
-from datajunction_server.construction.build import build_node, get_measures_query
+from datajunction_server.construction.build import (
+    build_node,
+    get_default_criteria,
+    get_measures_query,
+)
 from datajunction_server.database.materialization import Materialization
 from datajunction_server.database.node import NodeRevision
 from datajunction_server.errors import DJException, DJInvalidInputException
@@ -183,11 +187,15 @@ async def build_non_cube_materialization_config(
     """
     Build materialization config for non-cube nodes (transforms and dimensions).
     """
+    build_criteria = get_default_criteria(
+        node=current_revision,
+    )
     materialization_ast = await build_node(
         session=session,
         node=current_revision,
         dimensions=[],
         orderby=[],
+        build_criteria=build_criteria,
     )
     generic_config = GenericMaterializationConfig(
         lookback_window=upsert.config.lookback_window,

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/models/node.py

@@ -45,7 +45,7 @@ class BuildCriteria:
 
     timestamp: Optional[UTCDatetime] = None
     dialect: Dialect = Dialect.SPARK
-    for_materialization: bool = False
+    target_node_name: Optional[str] = None
 
 
 class NodeMode(StrEnum):

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/sql/functions.py

@@ -278,6 +278,8 @@ class Aggregate(Function):
             ast,
         )
 
+        # aggregate's lambda function can take three or four arguments, depending on whether
+        # an optional finish function is provided
         if len(args) == 4:
             expr, start, merge, finish = args
         else:

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/datajunction_server/sql/parsing/ast.py

@@ -1734,7 +1734,10 @@ class Function(Named, Operation):
         try:
             self.function().compile_lambda(*self.args)
         except DJParseException as parse_exc:
-            logger.warning(parse_exc)
+            if "Cannot resolve type of column" in parse_exc.message:
+                logger.warning(parse_exc)
+            else:
+                raise parse_exc
 
         for child in self.children:
             if not child.is_compiled():

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/tests/api/materializations_test.py

@@ -1058,3 +1058,150 @@ async def test_spark_sql_incremental(
             .replace("${country}", "DJ_COUNTRY()"),
         ),
     )
+
+
+@pytest.mark.asyncio
+async def test_spark_with_availablity(
+    client_with_repairs_cube: AsyncClient,  # pylint: disable=redefined-outer-name
+):
+    """
+    Verify that we build the Query correctly with or without the materialization availability.
+    """
+    # add one transform node
+    response = await client_with_repairs_cube.post(
+        "/nodes/transform/",
+        json={
+            "query": ("SELECT first_name, birth_date FROM default.hard_hats"),
+            "description": "test transform",
+            "mode": "published",
+            "name": "default.test_transform",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # add another transform node on top of the 1st one
+    response = await client_with_repairs_cube.post(
+        "/nodes/transform/",
+        json={
+            "query": ("SELECT first_name, birth_date FROM default.test_transform"),
+            "description": "test transform two",
+            "mode": "published",
+            "name": "default.test_transform_two",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # create a materialization on the 1st node (w/o availability)
+    response = await client_with_repairs_cube.post(
+        "/nodes/default.test_transform/materialization/",
+        json={
+            "job": "spark_sql",
+            "strategy": "full",
+            "config": {},
+            "schedule": "0 * * * *",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # check the materialization on 1st node
+    query_one = (
+        "SELECT  default_DOT_test_transform.first_name,"
+        "\n\tdefault_DOT_test_transform.birth_date "
+        "\n FROM (SELECT  default_DOT_hard_hats.first_name,"
+        "\n\tdefault_DOT_hard_hats.birth_date "
+        "\n FROM roads.hard_hats AS default_DOT_hard_hats)"
+        "\n AS default_DOT_test_transform\n\n"
+    )
+    response = await client_with_repairs_cube.get(
+        "/nodes/default.test_transform/materializations/",
+    )
+    assert response.status_code in (200, 201)
+    assert response.json()[0]["config"]["query"] == query_one
+
+    # create a materialization on the 2nd node (w/o availability)
+    response = await client_with_repairs_cube.post(
+        "/nodes/default.test_transform_two/materialization/",
+        json={
+            "job": "spark_sql",
+            "strategy": "full",
+            "config": {},
+            "schedule": "0 * * * *",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # check the materialization on 2nd node
+    query_two = (
+        "SELECT  default_DOT_test_transform_two.first_name,"
+        "\n\tdefault_DOT_test_transform_two.birth_date "
+        "\n FROM (SELECT  default_DOT_test_transform.first_name,"
+        "\n\tdefault_DOT_test_transform.birth_date "
+        "\n FROM (SELECT  default_DOT_hard_hats.first_name,"
+        "\n\tdefault_DOT_hard_hats.birth_date "
+        "\n FROM roads.hard_hats AS default_DOT_hard_hats)"
+        "\n AS default_DOT_test_transform)"
+        "\n AS default_DOT_test_transform_two\n\n"
+    )
+    response = await client_with_repairs_cube.get(
+        "/nodes/default.test_transform_two/materializations/",
+    )
+    assert response.status_code in (200, 201)
+    assert response.json()[0]["config"]["query"] == query_two
+
+    # add some availability to the 1st node
+    response = await client_with_repairs_cube.post(
+        "/data/default.test_transform/availability/",
+        json={
+            "catalog": "default",
+            "schema_": "accounting",
+            "table": "test_transform_materialized",
+            "valid_through_ts": 20230125,
+            "max_temporal_partition": ["2023", "01", "25"],
+            "min_temporal_partition": ["2022", "01", "01"],
+            "url": "http://some.catalog.com/default.accounting.pmts",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # refresh the materialization on 1st node now with availability (query should be the same)
+    response = await client_with_repairs_cube.post(
+        "/nodes/default.test_transform/materialization/",
+        json={
+            "job": "spark_sql",
+            "strategy": "full",
+            "config": {},
+            "schedule": "0 * * * *",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # check the materialization query again (query should be the same)
+    response = await client_with_repairs_cube.get(
+        "/nodes/default.test_transform/materializations/",
+    )
+    assert response.status_code in (200, 201)
+    assert response.json()[0]["config"]["query"] == query_one
+
+    # refresh the materialization on 2nd node now with availability
+    # (query should now include availability of the 1st node)
+    response = await client_with_repairs_cube.post(
+        "/nodes/default.test_transform_two/materialization/",
+        json={
+            "job": "spark_sql",
+            "strategy": "full",
+            "config": {},
+            "schedule": "0 * * * *",
+        },
+    )
+    assert response.status_code in (200, 201)
+
+    # check the materialization query again (query should be the same)
+    response = await client_with_repairs_cube.get(
+        "/nodes/default.test_transform_two/materializations/",
+    )
+    assert response.status_code in (200, 201)
+    assert response.json()[0]["config"]["query"] != query_two
+    assert (
+        "FROM accounting.test_transform_materialized "
+        in response.json()[0]["config"]["query"]
+    )

{datajunction_server-0.0.1a47.dev0 → datajunction_server-0.0.1a48}/tests/construction/build_test.py

@@ -7,11 +7,12 @@ from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 import datajunction_server.sql.parsing.types as ct
-from datajunction_server.construction.build import build_node
+from datajunction_server.construction.build import build_node, get_default_criteria
 from datajunction_server.database.attributetype import AttributeType, ColumnAttribute
 from datajunction_server.database.column import Column
 from datajunction_server.database.node import Node, NodeRevision
 from datajunction_server.errors import DJException
+from datajunction_server.models.engine import Dialect
 from datajunction_server.models.node_type import NodeType
 from datajunction_server.naming import amenable_name
 from datajunction_server.sql.parsing.backends.antlr4 import parse
@@ -278,3 +279,10 @@ async def test_build_node_with_unnamed_column(construction_session: AsyncSession
 def test_amenable_name():
     """testing for making an amenable name"""
     assert amenable_name("hello.名") == "hello_DOT__UNK"
+
+
+def test_get_default_criteria():
+    """Test getting default criteria for a node revision"""
+    result = get_default_criteria(node=NodeRevision(type=NodeType.TRANSFORM))
+    assert result.dialect == Dialect.SPARK
+    assert result.target_node_name is None

datajunction_server-0.0.1a47.dev0/datajunction_server/__about__.py

@@ -1,4 +0,0 @@
-"""
-Version for Hatch
-"""
-__version__ = "0.0.1a47.dev0"