datajunction-server 0.0.1a35.dev1__tar.gz → 0.0.1a36__tar.gz

datajunction_server-0.0.1a36/.env

@@ -0,0 +1,2 @@
+QUERY_SERVICE=http://djqs:8001
+SECRET=a-fake-secretkey

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-server
-Version: 0.0.1a35.dev1
+Version: 0.0.1a36
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

datajunction_server-0.0.1a36/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a36"

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/api/data.py

@@ -253,10 +253,20 @@ def get_data_for_metrics(  # pylint: disable=R0914, R0913
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     engine_name: Optional[str] = None,
     engine_version: Optional[str] = None,
+    current_user: Optional[User] = Depends(get_current_user),
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> QueryWithResults:
     """
     Return data for a set of metrics with dimensions and filters
     """
+    access_control = access.AccessControlStore(
+        validate_access=validate_access,
+        user=current_user,
+        base_verb=access.ResourceRequestVerb.READ,
+    )
+
     translated_sql, engine, catalog = build_sql_for_multiple_metrics(
         session,
         metrics,
@@ -266,6 +276,7 @@ def get_data_for_metrics(  # pylint: disable=R0914, R0913
         limit,
         engine_name,
         engine_version,
+        access_control,
     )
 
     query_create = QueryCreate(

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/api/helpers.py

@@ -651,6 +651,7 @@ def validate_cube(  # pylint: disable=too-many-locals
     session: Session,
     metric_names: List[str],
     dimension_names: List[str],
+    require_dimensions: bool = True,
 ) -> Tuple[List[Column], List[Node], List[Node], List[Column], Optional[Catalog]]:
     """
     Validate that a set of metrics and dimensions can be built together.
@@ -689,20 +690,20 @@ def validate_cube(  # pylint: disable=too-many-locals
 
     # Verify that the provided dimension attributes exist
     for dimension_attribute in dimension_names:
-        node_name, column_name = dimension_attribute.rsplit(".", 1)
         try:
+            node_name, column_name = dimension_attribute.rsplit(".", 1)
             dimension_node = get_node_by_name(session=session, name=node_name)
-        except DJNodeNotFound as exc:  # pragma: no cover
+        except (ValueError, DJNodeNotFound) as exc:  # pragma: no cover
             raise DJException(
-                f"{exc.message} Please make sure that `{column_name}` "
-                f"is an attribute on a dimension node `{node_name}`.",
+                f"Please make sure that `{dimension_attribute}` "
+                "is a dimensional attribute.",
             ) from exc
         dimension_nodes.append(dimension_node)
         columns = {col.name: col for col in dimension_node.current.columns}
         if column_name in columns:  # pragma: no cover
             dimensions.append(columns[column_name])
 
-    if not dimensions:
+    if require_dimensions and not dimensions:
         raise DJException(
             message=("At least one dimension is required"),
             http_status_code=http.client.UNPROCESSABLE_ENTITY,
@@ -817,6 +818,7 @@ def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,too-ma
         session,
         metrics,
         dimensions,
+        require_dimensions=False,
     )
     leading_metric_node = get_node_by_name(session, metrics[0])
     available_engines = leading_metric_node.current.catalog.engines

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/api/namespaces.py

@@ -3,7 +3,7 @@ Node namespace related APIs.
 """
 import logging
 from http import HTTPStatus
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 from fastapi import Depends, Query
 from fastapi.responses import JSONResponse
@@ -24,6 +24,8 @@ from datajunction_server.internal.access.authorization import (
 from datajunction_server.internal.namespaces import (
     create_namespace,
     get_nodes_in_namespace,
+    get_nodes_in_namespace_detailed,
+    get_project_config,
     hard_delete_namespace,
     mark_namespace_deactivated,
     mark_namespace_restored,
@@ -300,3 +302,22 @@ def hard_delete_node_namespace(
             "impact": impacts,
         },
     )
+
+
+@router.get(
+    "/namespaces/{namespace}/export/",
+    name="Export a namespace as a single project's metadata",
+)
+def export_a_namespace(
+    namespace: str,
+    *,
+    session: Session = Depends(get_session),
+) -> List[Dict]:
+    """
+    Generates a zip of YAML files for the contents of the given namespace
+    as well as a project definition file.
+    """
+    return get_project_config(
+        nodes=get_nodes_in_namespace_detailed(session, namespace),
+        namespace_requested=namespace,
+    )

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/construction/build.py

@@ -930,9 +930,10 @@ def build_metric_nodes(
         # bind the table for this built metric to all columns in the
         organization = cast(ast.Organization, parent_ast.select.organization)
         parent_ast.select.organization = None
-        for col in organization.find_all(ast.Column):
-            col.add_table(current_cte_as_table)  # pragma: no cover
-        orderby_sort_items += organization.order  # type: ignore
+        if organization:  # pragma: no cover
+            for col in organization.find_all(ast.Column):
+                col.add_table(current_cte_as_table)
+            orderby_sort_items += organization.order  # type: ignore
 
         final_select_columns = [
             ast.Column(

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/internal/namespaces.py

@@ -1,8 +1,9 @@
 """
 Helper methods for namespaces endpoints.
 """
+import os
 from datetime import datetime
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 
 from sqlalchemy import or_
 from sqlalchemy.sql.operators import is_
@@ -15,6 +16,7 @@ from datajunction_server.errors import (
     DJInvalidInputException,
 )
 from datajunction_server.models import History, User
+from datajunction_server.models.cube import CubeRevisionMetadata
 from datajunction_server.models.history import ActivityType, EntityType
 from datajunction_server.models.node import Node, NodeNamespace, NodeRevision, NodeType
 from datajunction_server.typing import UTCDatetime
@@ -56,6 +58,27 @@ def get_nodes_in_namespace(
     return session.exec(list_nodes_query).all()
 
 
+def get_nodes_in_namespace_detailed(
+    session: Session,
+    namespace: str,
+    node_type: NodeType = None,
+) -> List[Node]:
+    """
+    Gets a list of node names (w/ full details) in the namespace
+    """
+    get_node_namespace(session, namespace)
+    list_nodes_query = select(Node).where(
+        or_(
+            col(Node.namespace).like(f"{namespace}.%"),  # pylint: disable=no-member
+            Node.namespace == namespace,
+        ),
+        Node.current_version == NodeRevision.version,
+        Node.name == NodeRevision.name,
+        Node.type == node_type if node_type else True,
+    )
+    return session.exec(list_nodes_query).all()
+
+
 def list_namespaces_in_hierarchy(  # pylint: disable=too-many-arguments
     session: Session,
     namespace: str,
@@ -234,3 +257,175 @@ def hard_delete_namespace(
         }
         session.delete(_namespace)
     return impacts
+
+
+def _get_dir_and_filename(
+    node_name: str,
+    node_type: str,
+    namespace_requested: str,
+) -> Tuple[str, str]:
+    """
+    Get the directory and filename where a node name would be located
+    """
+    dot_split = node_name.replace(f"{namespace_requested}.", "").split(".")
+    filename = f"{dot_split[-1]}.{node_type}.yaml"
+    directory = os.path.sep.join(dot_split[:-1])
+    return filename, directory
+
+
+def _source_project_config(node: Node, namespace_requested: str) -> Dict:
+    """
+    Returns a project config definition for a source node
+    """
+    filename, directory = _get_dir_and_filename(
+        node_name=node.name,
+        node_type=node.type,
+        namespace_requested=namespace_requested,
+    )
+    return {
+        "filename": filename,
+        "directory": directory,
+        "description": node.current.description,
+        "table": f"{node.current.catalog}.{node.current.schema_}.{node.current.table}",
+        "columns": [
+            {"name": column.name, "type": str(column.type)}
+            for column in node.current.columns
+        ],
+        "dimension_links": {
+            column.name: {"dimension": column.dimension.name}
+            for column in node.current.columns
+            if column.dimension
+        },
+    }
+
+
+def _transform_project_config(node: Node, namespace_requested: str) -> Dict:
+    """
+    Returns a project config definition for a transform node
+    """
+    filename, directory = _get_dir_and_filename(
+        node_name=node.name,
+        node_type=node.type,
+        namespace_requested=namespace_requested,
+    )
+    return {
+        "filename": filename,
+        "directory": directory,
+        "description": node.current.description,
+        "query": node.current.query,
+        "dimension_links": {
+            column.name: {"dimension": column.dimension.name}
+            for column in node.current.columns
+            if column.dimension
+        },
+    }
+
+
+def _dimension_project_config(node: Node, namespace_requested: str) -> Dict:
+    """
+    Returns a project config definition for a dimension node
+    """
+    filename, directory = _get_dir_and_filename(
+        node_name=node.name,
+        node_type=node.type,
+        namespace_requested=namespace_requested,
+    )
+    return {
+        "filename": filename,
+        "directory": directory,
+        "description": node.current.description,
+        "query": node.current.query,
+        "primary_key": [pk.name for pk in node.current.primary_key()],
+        "dimension_links": {
+            column.name: {"dimension": column.dimension.name}
+            for column in node.current.columns
+            if column.dimension
+        },
+    }
+
+
+def _metric_project_config(node: Node, namespace_requested: str) -> Dict:
+    """
+    Returns a project config definition for a metric node
+    """
+    filename, directory = _get_dir_and_filename(
+        node_name=node.name,
+        node_type=node.type,
+        namespace_requested=namespace_requested,
+    )
+    return {
+        "filename": filename,
+        "directory": directory,
+        "description": node.current.description,
+        "query": node.current.query,
+    }
+
+
+def _cube_project_config(node: Node, namespace_requested: str) -> Dict:
+    """
+    Returns a project config definition for a cube node
+    """
+    filename, directory = _get_dir_and_filename(
+        node_name=node.name,
+        node_type=NodeType.CUBE,
+        namespace_requested=namespace_requested,
+    )
+    cube_revision = CubeRevisionMetadata.from_orm(node.current)
+    metrics = []
+    dimensions = []
+    for element in cube_revision.cube_elements:
+        if element.type == NodeType.METRIC:
+            metrics.append(element.node_name)
+        else:
+            dimensions.append(f"{element.node_name}.{element.name}")
+    return {
+        "filename": filename,
+        "directory": directory,
+        "description": cube_revision.description,
+        "metrics": metrics,
+        "dimensions": dimensions,
+    }
+
+
+def get_project_config(nodes: List[Node], namespace_requested: str) -> List[Dict]:
+    """
+    Returns a project config definition
+    """
+    project_components = []
+    for node in nodes:
+        if node.type == NodeType.SOURCE:
+            project_components.append(
+                _source_project_config(
+                    node=node,
+                    namespace_requested=namespace_requested,
+                ),
+            )
+        elif node.type == NodeType.TRANSFORM:
+            project_components.append(
+                _transform_project_config(
+                    node=node,
+                    namespace_requested=namespace_requested,
+                ),
+            )
+        elif node.type == NodeType.DIMENSION:
+            project_components.append(
+                _dimension_project_config(
+                    node=node,
+                    namespace_requested=namespace_requested,
+                ),
+            )
+        elif node.type == NodeType.METRIC:
+            project_components.append(
+                _metric_project_config(
+                    node=node,
+                    namespace_requested=namespace_requested,
+                ),
+            )
+        else:
+            project_components.append(
+                _cube_project_config(
+                    node=node,
+                    namespace_requested=namespace_requested,
+                ),
+            )
+    return project_components

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/models/node.py

@@ -1127,6 +1127,9 @@ class DimensionAttributeOutput(SQLModel):
     """
 
     name: str
+    node_name: Optional[str]
+    node_display_name: Optional[str]
+    is_primary_key: bool
     type: ColumnType
     path: List[str]
 

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/sql/dag.py

@@ -73,6 +73,10 @@ def get_dimensions(
                 dimensions.append(
                     DimensionAttributeOutput(
                         name=f"{current_node.name}.{column.name}",
+                        node_name=current_node.current.name,
+                        node_display_name=current_node.current.display_name,
+                        is_primary_key=column.name
+                        in {pk.name for pk in current_node.current.primary_key()},
                         type=column.type,
                         path=join_path_str,
                     ),

{datajunction_server-0.0.1a35.dev1 → datajunction_server-0.0.1a36}/datajunction_server/sql/parsing/ast.py

@@ -1365,6 +1365,7 @@ class BinaryOpKind(DJEnum):
     Plus = "+"
     Minus = "-"
     Modulo = "%"
+    NullSafeEq = "<=>"
 
 
 @dataclass(eq=False)
@@ -2174,7 +2175,7 @@ class FunctionTable(FunctionTableExpression):
             if self.column_list
             else ""
         )
-        column_list_str = f"({cols})" if alias and cols else ""
+        column_list_str = f"({cols})" if cols else ""
         args_str = f"({', '.join(str(col) for col in self.args)})" if self.args else ""
         return f"{self.name}{args_str}{as_}{alias}{column_list_str}"
 
@@ -2555,9 +2556,3 @@ class Query(TableExpression, UnNamed):
             access_control,
         )
         self.select.add_aliases_to_unnamed_columns()
-
-        # Make the generated query deterministic
-        self.select.projection = sorted(
-            self.select.projection,
-            key=lambda x: str(x.alias_or_name),
-        )[:]