datajunction-server 0.0.2.dev4__py3-none-any.whl → 0.0.2.dev5__py3-none-any.whl

datajunction_server/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.2.dev4"
+__version__ = "0.0.2.dev5"

datajunction_server/database/node.py

@@ -390,8 +390,11 @@ class Node(Base):
                 for col in self.current.columns
                 if col.dimension_id and col.dimension_column
             ]
+            col_specs = [col.to_spec() for col in self.current.columns]
             extra_kwargs.update(
-                primary_key=[col.name for col in self.current.primary_key()],
+                primary_key=[
+                    col.name for col in col_specs if "primary_key" in col.attributes
+                ],
                 dimension_links=join_link_specs + ref_link_specs,
             )
 
@@ -438,15 +441,6 @@ class Node(Base):
         node_spec_cls = node_spec_class_map[self.type]
         return node_spec_cls(**base_kwargs, **extra_kwargs)
 
-    @classmethod
-    def default_load_options(cls) -> List[ExecutableOption]:
-        return [
-            joinedload(Node.current).options(*NodeRevision.default_load_options()),
-            selectinload(Node.tags),
-            selectinload(Node.created_by),
-            selectinload(Node.owners),
-        ]
-
     @classmethod
     def cube_load_options(cls) -> List[ExecutableOption]:
         return [

datajunction_server/internal/deployment.py

@@ -51,6 +51,7 @@ from datajunction_server.internal.nodes import (
     create_a_cube,
     create_a_node,
     create_a_source_node,
+    hard_delete_node,
     refresh_source,
     remove_dimension_link,
     set_node_column_attributes,
@@ -103,18 +104,6 @@ async def safe_task(
         )
 
 
-async def find_existing_nodes(
-    session: AsyncSession,
-    nodes: list[NodeSpec],
-) -> dict[str, NodeSpec]:
-    non_cubes = [n.rendered_name for n in nodes if n.node_type != NodeType.CUBE]
-    cubes = [n.rendered_name for n in nodes if n.node_type == NodeType.CUBE]
-    all_nodes = await Node.get_by_names(session, non_cubes) + [
-        await Node.get_cube_by_name(session, cube) for cube in cubes
-    ]
-    return {node.name: await node.to_spec(session) for node in all_nodes if node}
-
-
 async def deploy(
     deployment_id: str,
     deployment: DeploymentSpec,
@@ -149,8 +138,6 @@ async def deploy(
             current_user,
             save_history,
         )
-
-        # async with session_context(request) as session:
         all_nodes = await NodeNamespace.list_all_nodes(
             session,
             deployment.namespace,
@@ -174,7 +161,7 @@ async def deploy(
             for node_spec in to_skip
         ],
     )
-    if not to_deploy:
+    if not to_deploy and not to_delete:
         logger.info(
             "No changes detected, skipping deployment. Total elapsed: %.3fs",
             time.perf_counter() - start_total,
@@ -182,9 +169,10 @@ async def deploy(
         return deployed_results
 
     logger.info(
-        "Found %d nodes to deploy; skipped %d nodes",
+        "Found %d nodes to deploy, skipped %d nodes, deleting %d nodes",
         len(to_deploy),
         len(to_skip),
+        len(to_delete),
     )
 
     node_graph = extract_node_graph(
@@ -255,9 +243,16 @@ async def deploy(
         DeploymentStatus.RUNNING,
         deployed_results,
     )
+
     logger.info("Starting deletion of %d nodes", len(to_delete))
-    # for node in to_delete:
-    #     await deactivate_node(session, node.name, current_user, save_history)
+    for node_spec in to_delete:
+        deployed_results.append(
+            await deploy_delete_node(
+                node_name=node_spec.rendered_name,
+                current_username=current_username,
+                save_history=save_history,
+            ),
+        )
     logger.info("Finished deploying namespace %s", deployment.namespace)
     return deployed_results
 
@@ -417,6 +412,11 @@ def filter_nodes_to_deploy(
         len(to_skip),
         [result.rendered_name for result in to_skip],
     )
+    logger.info(
+        "Deleting %d nodes: %s",
+        len(to_delete),
+        to_delete,
+    )
     return to_create + to_update, to_skip, to_delete
 
 
@@ -565,7 +565,6 @@ async def deploy_links_for_node(
         deployment_result = await deploy_dimension_link_from_spec(
             node_spec=node_spec,
             link_spec=link,
-            request=request,
             current_username=current_username,
             save_history=save_history,
             existing_node_links=existing_node_links,
@@ -662,17 +661,17 @@ async def run_tasks_with_semaphore(
     )
 
 
-async def deploy_node_tags(node_name: str, node_spec: NodeSpec):
+async def deploy_node_tags(node_name: str, tag_names: list[str]) -> None:
     async with session_context() as session:
         node = await Node.get_by_name(session=session, name=node_name)
-        tags = await get_tags_by_name(session, names=node_spec.tags or [])
+        tags = await get_tags_by_name(session, names=tag_names or [])
         node.tags = tags  # type: ignore
         session.add(node)
         await session.commit()
         await session.refresh(node)
 
 
-async def deploy_column_attributes(
+async def deploy_column_properties(
     node_name: str,
     node_spec: NodeSpec,
     current_username: str,
@@ -685,30 +684,43 @@ async def deploy_column_attributes(
         desired_column_state = {col.name: col for col in node_spec.columns or []}
         for col in node.current.columns:  # type: ignore
             if desired_col := desired_column_state.get(col.name):
-                # If the column is explicitly defined, update its properties to match
-                if col.display_name != desired_col.display_name:
+                # Set column display name and description
+                if (
+                    col.display_name != desired_col.display_name
+                    and desired_col.display_name is not None
+                ):
                     col.display_name = desired_col.display_name
                     changed_columns.add(col.name)
                 if col.description != desired_col.description:
                     col.description = desired_col.description
                     changed_columns.add(col.name)
-                if desired_col.partition != (
-                    col.partition.to_spec() if col.partition else None
-                ):
-                    partition = (
-                        Partition(
-                            column_id=col.id,
-                            type_=desired_col.partition.type,
-                            format=desired_col.partition.format,
-                            granularity=desired_col.partition.granularity,
-                        )
-                        if desired_col.partition
-                        else None
+
+                # Set column partition
+                if desired_col.partition is None and col.partition:
+                    session.delete(col.partition)
+                    changed_columns.add(col.name)
+                elif col.partition is None and desired_col.partition:
+                    partition = Partition(
+                        column_id=col.id,
+                        type_=desired_col.partition.type,
+                        format=desired_col.partition.format,
+                        granularity=desired_col.partition.granularity,
                     )
-                    if partition:
-                        session.add(partition)
+                    session.add(partition)
                     col.partition = partition
                     changed_columns.add(col.name)
+                elif (
+                    desired_col.partition
+                    and col.partition
+                    and desired_col.partition != col.partition.to_spec()
+                ):
+                    col.partition.type_ = desired_col.partition.type
+                    col.partition.format = desired_col.partition.format
+                    col.partition.granularity = desired_col.partition.granularity
+                    session.add(col)
+                    changed_columns.add(col.name)
+
+                # Set column attributes
                 if set(desired_col.attributes) != set(col.attribute_names()):
                     await set_node_column_attributes(
                         session=session,
@@ -717,6 +729,11 @@ async def deploy_column_attributes(
                         attributes=[
                             AttributeTypeIdentifier(name=attr)
                             for attr in desired_col.attributes
+                        ]
+                        + [
+                            AttributeTypeIdentifier(name=attr)
+                            for attr in col.attribute_names()
+                            if attr == "primary_key"
                         ],
                         current_user=current_user,
                         save_history=save_history,
@@ -726,8 +743,14 @@ async def deploy_column_attributes(
                 # If the column is not explicitly defined, reset it to default
                 col.display_name = labelize(col.name)
                 col.description = ""
-                col.partition = None
-                col.attributes = []
+                if col.partition:
+                    session.delete(col.partition)
+                col.attributes = [
+                    attr
+                    for attr in col.attributes
+                    if attr.attribute_type.name == "primary_key"
+                ]
+                # col.attributes = [AttributeTypeIdentifier(name=attr) for attr in col.attributes if attr == "primary_key"]
 
             session.add(col)
         await session.commit()
@@ -787,7 +810,7 @@ async def deploy_node_from_spec(
         ) if changed_fields else ""
 
         if set(node_spec.tags) != set([tag.name for tag in node.tags]):
-            await deploy_node_tags(node_name=node.name, node_spec=node_spec)
+            await deploy_node_tags(node_name=node.name, tag_names=node_spec.tags)
             tags_list = ", ".join([f"`{tag}`" for tag in node_spec.tags])
             changelog.append(f"└─ Set tags to {tags_list}.")
         if node.type in (
@@ -796,7 +819,7 @@ async def deploy_node_from_spec(
             NodeType.DIMENSION,
             NodeType.CUBE,
         ):
-            changed_columns = await deploy_column_attributes(
+            changed_columns = await deploy_column_properties(
                 node_name=node.name,
                 node_spec=node_spec,
                 current_username=current_username,
@@ -963,6 +986,12 @@ async def deploy_transform_dimension_node_from_spec(
             save_history=save_history,
             cache=cache,
         )
+        created_node = await Node.get_by_name(  # type: ignore
+            session,
+            node_spec.rendered_name,
+            options=NodeOutput.load_options(),
+            raise_if_not_exists=True,
+        )
         return created_node
 
 
@@ -1123,7 +1152,6 @@ async def deploy_cube_node_from_spec(
 async def deploy_dimension_link_from_spec(
     node_spec: NodeSpec,
     link_spec: DimensionLinkSpec,
-    request: Request,
     current_username: str,
     save_history: Callable,
     existing_node_links: dict[
@@ -1138,13 +1166,15 @@ async def deploy_dimension_link_from_spec(
             if link_spec.rendered_dimension_node not in existing_node_links
             else DeploymentResult.Operation.UPDATE
         )
-        async with session_context(request) as session:
+        async with session_context() as session:
             current_user = cast(
                 User,
                 await User.get_by_username(session, current_username),
             )
             if link_spec.type == LinkType.JOIN:
-                existing = existing_node_links.get(link_spec.rendered_dimension_node)
+                existing = existing_node_links.get(
+                    (link_spec.rendered_dimension_node, link_spec.role),
+                )
                 join_link = cast(DimensionJoinLinkSpec, link_spec)
                 if existing != join_link:
                     if join_link.node_column:
@@ -1240,11 +1270,43 @@ async def deploy_remove_dimension_link(
             status=DeploymentResult.Status.SUCCESS,
             operation=DeploymentResult.Operation.DELETE,
         )
-    except Exception as exc:  # pragma: no cover
-        return DeploymentResult(  # pragma: no cover
+    except Exception as exc:
+        return DeploymentResult(
             deploy_type=DeploymentResult.Type.LINK,
             name=f"{node_name} -> {link.rendered_dimension_node}",
             status=DeploymentResult.Status.FAILED,
             operation=DeploymentResult.Operation.DELETE,
             message=str(exc),
         )
+
+
+async def deploy_delete_node(
+    node_name: str,
+    current_username: str,
+    save_history: Callable,
+) -> DeploymentResult:
+    async with session_context() as session:
+        current_user = cast(User, await User.get_by_username(session, current_username))
+        try:
+            await hard_delete_node(
+                name=node_name,
+                session=session,
+                current_user=current_user,
+                save_history=save_history,
+            )
+            return DeploymentResult(
+                name=node_name,
+                deploy_type=DeploymentResult.Type.NODE,
+                status=DeploymentResult.Status.SUCCESS,
+                operation=DeploymentResult.Operation.DELETE,
+                message=f"Node {node_name} has been removed.",
+            )
+        except Exception as exc:
+            logger.exception(exc)
+            return DeploymentResult(
+                name=node_name,
+                deploy_type=DeploymentResult.Type.NODE,
+                status=DeploymentResult.Status.FAILED,
+                operation=DeploymentResult.Operation.DELETE,
+                message=str(exc),
+            )

datajunction_server/internal/nodes.py

@@ -2746,7 +2746,7 @@ async def hard_delete_node(
         name,
         options=[joinedload(Node.current), joinedload(Node.revisions)],
         include_inactive=True,
-        raise_if_not_exists=False,
+        raise_if_not_exists=True,
     )
     downstream_nodes = await get_downstream_nodes(session=session, node_name=name)
 

datajunction_server/models/deployment.py

@@ -5,6 +5,7 @@ from typing import Any, Literal, Union
 
 from datajunction_server.models.partition import Granularity, PartitionType
 from datajunction_server.errors import DJInvalidInputException
+from datajunction_server.models.base import labelize
 from datajunction_server.models.dimensionlink import JoinType, LinkType
 from datajunction_server.models.node import (
     MetricDirection,
@@ -291,11 +292,6 @@ class LinkableNodeSpec(NodeSpec):
             other.dimension_links or [],
             key=lambda link: link.rendered_dimension_node,
         )
-        print(
-            "Comparing LinkableNodeSpec",
-            self.rendered_name,
-            eq_columns(self.columns, other.columns),
-        )
         return (
             super().__eq__(other)
             and eq_columns(self.columns, other.columns)
@@ -434,11 +430,6 @@ class CubeSpec(NodeSpec):
         ]
 
     def __eq__(self, other: Any) -> bool:
-        print(
-            "Comparing CubeSpec",
-            self.rendered_name,
-            eq_columns(self.columns, other.columns),
-        )
         return (
             super().__eq__(other)
             and eq_columns(self.columns, other.columns)
@@ -575,25 +566,38 @@ def eq_columns(a: list[ColumnSpec] | None, b: list[ColumnSpec] | None) -> bool:
       - None or [] is considered equivalent to a list where every column only has 'primary_key'
         in attributes and partition is None.
     """
-    a_list = a or []
-    b_list = b or []
-
-    a_map = {col.name: col for col in a_list}
-    b_map = {col.name: col for col in b_list}
-    for col_name, col_a in a_map.items():
-        col_b = b_map.get(col_name)
-        if (set(col_a.attributes if col_a else []) - {"primary_key"}) != (  # type: ignore
-            set(col_b.attributes if col_b else []) - {"primary_key"}  # type: ignore
-        ) or (col_a.partition if col_a else None) != (
-            col_b.partition if col_b else None
-        ):  # type: ignore
-            return False
-    for col_name, col_b in b_map.items():
-        col_a = a_map.get(col_name)  # type: ignore
-        if (set(col_b.attributes if col_b else []) - {"primary_key"}) != (  # type: ignore
-            set(col_a.attributes if col_a else []) - {"primary_key"}  # type: ignore
-        ) or (col_b.partition if col_b else None) != (
-            col_a.partition if col_a else None
-        ):  # type: ignore
-            return False
-    return True
+    a_map = {col.name: col for col in a or []}
+    b_map = {col.name: col for col in b or []}
+    a_cols, b_cols = [], []
+    for col_name in set(a_map.keys()).union(set(b_map.keys())):
+        a_col = a_map.get(col_name)
+        b_col = b_map.get(col_name)
+        if not a_col:
+            a_col = ColumnSpec(
+                name=col_name,
+                display_name=labelize(col_name),
+                type=b_col.type if b_col else "",
+                attributes=[],
+            )
+        if not a_col.display_name:
+            a_col.display_name = labelize(col_name)
+        if not a_col.description:
+            a_col.description = ""
+        if not b_col:
+            b_col = ColumnSpec(
+                name=col_name,
+                display_name=labelize(col_name),
+                type=a_col.type if a_col else "",
+                attributes=[],
+            )
+        if not b_col.display_name:
+            b_col.display_name = labelize(col_name)
+        if not b_col.description:
+            b_col.description = ""
+        if "primary_key" in a_col.attributes:
+            a_col.attributes = list(set(a_col.attributes) - {"primary_key"})
+        if "primary_key" in b_col.attributes:
+            b_col.attributes = list(set(b_col.attributes) - {"primary_key"})
+        a_cols.append(a_col)
+        b_cols.append(b_col)
+    return a_cols == b_cols

{datajunction_server-0.0.2.dev4.dist-info → datajunction_server-0.0.2.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction-server
-Version: 0.0.2.dev4
+Version: 0.0.2.dev5
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: Homepage, https://datajunction.io
 Project-URL: Repository, https://github.com/DataJunction/dj

{datajunction_server-0.0.2.dev4.dist-info → datajunction_server-0.0.2.dev5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-datajunction_server/__about__.py,sha256=7_8nP1iwEx_vfBpO-HcgSpLMpKEmeBk1LgGxMXjgACM,54
+datajunction_server/__about__.py,sha256=s6aBpR8toU4qlGgMXWyqsgZXfKeMKdKuQ7QP7uwKle8,54
 datajunction_server/__init__.py,sha256=nN5-uJoSVEwuc8n-wMygqeF0Xhxi_zqqbCgutZvAt3E,384
 datajunction_server/alembic.ini,sha256=mclJ_xx8pHfRyZ69SA9ZPqUwZaaQCTyxZ6wBmbrf1bo,3024
 datajunction_server/config.py,sha256=L1zkaiF82S-ciR-wVeILx7CWKSOPPJ90a9zooXVNHEc,5641
@@ -128,7 +128,7 @@ datajunction_server/database/materialization.py,sha256=7_w5ay2RXukTaRS03UILsMFrN
 datajunction_server/database/measure.py,sha256=iwZQIhM-_OlZ4v2a3BmZrhpC4ej-MGh6rLqGB70XGYo,6387
 datajunction_server/database/metricmetadata.py,sha256=jKh58lOf5_MoFyDTVtpWhK9-Ox22ryWK2XtBKjPApc4,2084
 datajunction_server/database/namespace.py,sha256=T5GF4eDLgw51xdjWC0T5G_mcAgL4E7OrYuk6x552dhM,5685
-datajunction_server/database/node.py,sha256=4BJPUaoYQGz-CtBHPcSB8-I1S4yzkd8-Q2NOvRAtYpk,44951
+datajunction_server/database/node.py,sha256=M9a0BcBvhVtIKrOBxDoBECRzsO8tXkfTagnywjqWgv8,44770
 datajunction_server/database/nodeowner.py,sha256=YydIO1F-S-J6qyNW8ui71H-l6aENbW7RhdX5koIBq8E,1196
 datajunction_server/database/notification_preference.py,sha256=uLy60fWLWLJE940Qj7bY0Rq5b5flEkgcq2uxx4pRvPE,1892
 datajunction_server/database/partition.py,sha256=312KyT0IbaKk-lXxMRvAphJDfA83Wm1RsGp7-dT-U0g,5242
@@ -138,12 +138,12 @@ datajunction_server/database/user.py,sha256=-zW5ssf2IWgSjvZbOljOzVoqynBtNp3gQxHm
 datajunction_server/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datajunction_server/internal/client.py,sha256=hpManY1PTGQxAzx4S-1aUAoMvlWfPPU1wgL9iLlAg7o,11442
 datajunction_server/internal/cube_materializations.py,sha256=SJYIZfjAQr8n-pgECKoroGfOhoR_J0vnMmpCQ0mbLlY,10695
-datajunction_server/internal/deployment.py,sha256=N9p5kVllXC61yLX6JXiXRV2P_mjeIt-4gO8fWXm_cVE,45154
+datajunction_server/internal/deployment.py,sha256=snlmwkoWF0tUvIRpMYIhqpw5PYm8uEaqUYcXj1UGWcY,47416
 datajunction_server/internal/engines.py,sha256=RKIYIYJQMjRIFS7YXIW-q6vLUxilc3sd3eWHwTmDDu0,858
 datajunction_server/internal/history.py,sha256=MAj2ZTUCGgSLCaS1uB5dWtTqvb2SE_5HHLo0RReCRAc,816
 datajunction_server/internal/materializations.py,sha256=HVc1RPiarOBksvzyLP8Mt5GFvyEPlOqgIe151I_ZZMY,18184
 datajunction_server/internal/namespaces.py,sha256=Ff020CvpzWeypqReU5sPl3kNbHj1aQpw_6MI2DJhFSw,20713
-datajunction_server/internal/nodes.py,sha256=BbXzMg5G2Tk-xXcBD73UhWR7MQh3B_UeZ9r83LkK8tg,103715
+datajunction_server/internal/nodes.py,sha256=tTS4_oEgtSdf0PVorQ4toEr3iNyIrR8Q0AH4A9fLmWQ,103714
 datajunction_server/internal/notifications.py,sha256=8aP8KBs8k4kKXnQXIUPWyHa4uKsyV0oERipyP0LR0bs,1724
 datajunction_server/internal/seed.py,sha256=mc2TQP4dMwAyWTfGKj5b6wK6pvVyU6azHfxSozPElY0,2363
 datajunction_server/internal/sql.py,sha256=bboANhvtKufVYOOI6RF719GXQFWv-toVXIQQtKIFCtA,18475
@@ -185,7 +185,7 @@ datajunction_server/models/column.py,sha256=3TCa9dDAb9Q2WEAzpcdqAjKSX3PutbwlS8qG
 datajunction_server/models/cube.py,sha256=p6KmqoOVGlziH5k0wXs6qBvgWWQs984Ho6SGplDbtAQ,4459
 datajunction_server/models/cube_materialization.py,sha256=ydMRepDM95b4BBVay_nOYlRtK7OFG3pCDDTRdHe2awU,15787
 datajunction_server/models/database.py,sha256=xhCllbq5ikFNnrPvzuchxQUC9RWogzh73Eqz7Jj0i38,499
-datajunction_server/models/deployment.py,sha256=ZCRgGVy_FLHeoLc98-zRz_n79IZAPsr073Y8k4SOhqA,18051
+datajunction_server/models/deployment.py,sha256=DDK-vEoWJWTGX_761kzFtgH3uk4m20jQfxDOSf9O67U,18185
 datajunction_server/models/dialect.py,sha256=uifriawtKR4vi-GG2rdp4eeQkAkD6iLNuarCLQanTxY,3825
 datajunction_server/models/dimensionlink.py,sha256=WaCarxhawOiQqtH1EVS0RQRu9D8Bx6GFonLsdSVgXSs,1551
 datajunction_server/models/engine.py,sha256=Ebuy4HLkURr7mj0pxTj_Y4fEmF0oDen393rnj9fBnR0,466
@@ -226,7 +226,7 @@ datajunction_server/sql/parsing/backends/grammar/generated/SqlBaseParser.py,sha2
 datajunction_server/sql/parsing/backends/grammar/generated/SqlBaseParser.tokens,sha256=JDrzbaKDwIaimAZPYIUzCgzkOEgq0X5-a6_lz78lqgs,8131
 datajunction_server/sql/parsing/backends/grammar/generated/SqlBaseParserListener.py,sha256=vp8wduYkB-T5Xr6HZCSdzAxTHHPrDI5UJZXRJSVhAGA,102464
 datajunction_server/sql/parsing/backends/grammar/generated/SqlBaseParserVisitor.py,sha256=w3V03LgPIHCiqojNyuekBDYqskjOKlKrd0sczQAB_WQ,60290
-datajunction_server-0.0.2.dev4.dist-info/METADATA,sha256=h0bsPkDT6voeOPRcczUx6yrmvivISRd09p51Kq40Qq4,3769
-datajunction_server-0.0.2.dev4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-datajunction_server-0.0.2.dev4.dist-info/entry_points.txt,sha256=MOInJGdcQ10bDEl-XW4UMokEgx-ypINqBhObeDI8KiQ,74
-datajunction_server-0.0.2.dev4.dist-info/RECORD,,
+datajunction_server-0.0.2.dev5.dist-info/METADATA,sha256=wMmDeSs0m497HHgebVNxT01FnVOg9esWyRtkzfvbLx8,3769
+datajunction_server-0.0.2.dev5.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+datajunction_server-0.0.2.dev5.dist-info/entry_points.txt,sha256=MOInJGdcQ10bDEl-XW4UMokEgx-ypINqBhObeDI8KiQ,74
+datajunction_server-0.0.2.dev5.dist-info/RECORD,,