datajunction-server 0.0.1a35__tar.gz → 0.0.1a35.dev1__tar.gz

datajunction_server-0.0.1a35.dev1/.env

@@ -0,0 +1,3 @@
+QUERY_SERVICE=http://host.docker.internal:8002
+INDEX=sqlite:///dj.internal.db?check_same_thread=False
+SECRET=a-fake-secretkey

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/.gitignore

@@ -117,3 +117,6 @@ djqs.db
 
 # oauth credentials
 client_secret*
+
+# random notebooks
+Untitled*

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-server
-Version: 0.0.1a35
+Version: 0.0.1a35.dev1
 Summary: DataJunction server library for running to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>
@@ -22,6 +22,7 @@ Requires-Dist: cachelib<1.0.0,>=0.10.2
 Requires-Dist: cachetools>=5.3.1
 Requires-Dist: celery<6.0.0,>=5.2.7
 Requires-Dist: cryptography>=41.0.3
+Requires-Dist: fastapi-cache2>=0.2.1
 Requires-Dist: fastapi<0.80.0,>=0.79.0
 Requires-Dist: google-api-python-client>=2.95.0
 Requires-Dist: google-auth-httplib2>=0.1.0

datajunction_server-0.0.1a35.dev1/alembic/versions/2023_11_21_1611-d1a8716cf2d2_add_materialization_strategy_and_job_.py

@@ -0,0 +1,69 @@
+"""Add materialization strategy and job type
+
+Revision ID: d1a8716cf2d2
+Revises: 87cf9c00ac00
+Create Date: 2023-11-21 16:11:01.741721+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+import sqlmodel
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d1a8716cf2d2"
+down_revision = "87cf9c00ac00"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("materialization", schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                "strategy",
+                sa.Enum(
+                    "FULL",
+                    "SNAPSHOT",
+                    "INCREMENTAL_TIME",
+                    "VIEW",
+                    name="materializationstrategy",
+                ),
+                nullable=True,
+            ),
+        )
+        # batch_op.drop_constraint("node_revision_engine_uniq", type_="unique")
+        batch_op.create_unique_constraint(
+            "name_node_revision_uniq",
+            ["name", "node_revision_id"],
+        )
+        batch_op.drop_constraint(
+            "fk_materialization_engine_id_engine",
+            type_="foreignkey",
+        )
+        batch_op.drop_column("engine_id")
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("materialization", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("engine_id", sa.INTEGER(), nullable=False))
+        batch_op.create_foreign_key(
+            "fk_materialization_engine_id_engine",
+            "engine",
+            ["engine_id"],
+            ["id"],
+        )
+        batch_op.drop_constraint("name_node_revision_uniq", type_="unique")
+        batch_op.create_unique_constraint(
+            "node_revision_engine_uniq",
+            ["name", "node_revision_id", "engine_id"],
+        )
+        batch_op.drop_column("strategy")
+
+    # ### end Alembic commands ###

datajunction_server-0.0.1a35.dev1/datajunction_server/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a35.dev1"

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/catalogs.py

@@ -21,6 +21,8 @@ _logger = logging.getLogger(__name__)
 settings = get_settings()
 router = SecureAPIRouter(tags=["catalogs"])
 
+UNKNOWN_CATALOG_ID = 0
+
 
 @router.get("/catalogs/", response_model=List[CatalogInfo])
 def list_catalogs(*, session: Session = Depends(get_session)) -> List[CatalogInfo]:
@@ -120,3 +122,19 @@ def list_new_engines(
         if not already_set:
             new_engines.append(engine)
     return new_engines
+
+
+def default_catalog(session: Session = Depends(get_session)):
+    """
+    Loads a default catalog for nodes that are pure SQL and don't belong in any
+    particular catalog. This typically applies to on-the-fly user-defined dimensions.
+    """
+    statement = select(Catalog).filter(Catalog.id == UNKNOWN_CATALOG_ID)
+    catalogs = session.exec(statement).all()
+    if not catalogs:
+        unknown = Catalog(
+            id=UNKNOWN_CATALOG_ID,
+            name="unknown",
+        )
+        session.add(unknown)
+        session.commit()

datajunction_server-0.0.1a35.dev1/datajunction_server/api/cubes.py

@@ -0,0 +1,155 @@
+"""
+Cube related APIs.
+"""
+import logging
+from typing import List, Optional
+
+from fastapi import Depends, Query
+from sqlmodel import Session
+
+from datajunction_server.api.helpers import get_catalog_by_name, get_node_by_name
+from datajunction_server.construction.dimensions import build_dimensions_from_cube_query
+from datajunction_server.internal.access.authentication.http import SecureAPIRouter
+from datajunction_server.internal.access.authorization import validate_access
+from datajunction_server.models import access
+from datajunction_server.models.cube import (
+    CubeRevisionMetadata,
+    DimensionValue,
+    DimensionValues,
+)
+from datajunction_server.models.metric import TranslatedSQL
+from datajunction_server.models.node import NodeType
+from datajunction_server.models.query import QueryCreate
+from datajunction_server.service_clients import QueryServiceClient
+from datajunction_server.utils import (
+    from_amenable_name,
+    get_query_service_client,
+    get_session,
+    get_settings,
+)
+
+_logger = logging.getLogger(__name__)
+settings = get_settings()
+router = SecureAPIRouter(tags=["cubes"])
+
+
+@router.get("/cubes/{name}/", response_model=CubeRevisionMetadata, name="Get a Cube")
+def get_cube(
+    name: str, *, session: Session = Depends(get_session)
+) -> CubeRevisionMetadata:
+    """
+    Get information on a cube
+    """
+    node = get_node_by_name(session=session, name=name, node_type=NodeType.CUBE)
+    return node.current
+
+
+@router.get("/cubes/{name}/dimensions/sql", name="Dimensions SQL for Cube")
+def get_cube_dimension_sql(
+    name: str,
+    *,
+    dimensions: List[str] = Query([], description="Dimensions to get values for"),
+    filters: Optional[str] = Query(
+        None,
+        description="Filters on dimensional attributes",
+    ),
+    limit: Optional[int] = Query(
+        None,
+        description="Number of rows to limit the data retrieved to",
+    ),
+    include_counts: bool = False,
+    session: Session = Depends(get_session),
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=redefined-outer-name
+        validate_access,
+    ),
+) -> TranslatedSQL:
+    """
+    Generates SQL to retrieve all unique values of a dimension for the cube
+    """
+    node = get_node_by_name(session=session, name=name, node_type=NodeType.CUBE)
+    cube = node.current
+    return build_dimensions_from_cube_query(
+        session,
+        cube,
+        dimensions,
+        filters,
+        limit,
+        include_counts,
+        validate_access=validate_access,
+    )
+
+
+@router.get(
+    "/cubes/{name}/dimensions/data",
+    name="Dimensions Values for Cube",
+)
+def get_cube_dimension_values(  # pylint: disable=too-many-locals
+    name: str,
+    *,
+    dimensions: List[str] = Query([], description="Dimensions to get values for"),
+    filters: Optional[str] = Query(
+        None,
+        description="Filters on dimensional attributes",
+    ),
+    limit: Optional[int] = Query(
+        None,
+        description="Number of rows to limit the data retrieved to",
+    ),
+    include_counts: bool = False,
+    async_: bool = False,
+    session: Session = Depends(get_session),
+    query_service_client: QueryServiceClient = Depends(get_query_service_client),
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=redefined-outer-name
+        validate_access,
+    ),
+) -> DimensionValues:
+    """
+    All unique values of a dimension from the cube
+    """
+    node = get_node_by_name(session=session, name=name, node_type=NodeType.CUBE)
+    cube = node.current
+    translated_sql = build_dimensions_from_cube_query(
+        session,
+        cube,
+        dimensions,
+        filters,
+        limit,
+        include_counts,
+        validate_access,
+    )
+    if cube.availability:
+        catalog = get_catalog_by_name(  # pragma: no cover
+            session,
+            cube.availability.catalog,  # type: ignore
+        )
+    else:
+        catalog = cube.catalog
+    query_create = QueryCreate(
+        engine_name=catalog.engines[0].name,
+        catalog_name=catalog.name,
+        engine_version=catalog.engines[0].version,
+        submitted_query=translated_sql.sql,
+        async_=async_,
+    )
+    result = query_service_client.submit_query(query_create)
+    count_column = [
+        idx
+        for idx, col in enumerate(translated_sql.columns)  # type: ignore
+        if col.name == "count"
+    ]
+    dimension_values = [  # pragma: no cover
+        DimensionValue(
+            value=row[0 : count_column[0]] if count_column else row,
+            count=row[count_column[0]] if count_column else None,
+        )
+        for row in result.results.__root__[0].rows
+    ]
+    return DimensionValues(  # pragma: no cover
+        dimensions=[
+            from_amenable_name(col.name)
+            for col in translated_sql.columns  # type: ignore # pylint: disable=not-an-iterable
+            if col.name != "count"
+        ],
+        values=dimension_values,
+        cardinality=len(dimension_values),
+    )

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/data.py

@@ -195,7 +195,7 @@ def get_data(  # pylint: disable=too-many-locals
     )
 
     columns = [
-        assemble_column_metadata(col, node_name)  # type: ignore
+        assemble_column_metadata(col)  # type: ignore
         for col in query_ast.select.projection
     ]
 

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/helpers.py

@@ -23,6 +23,7 @@ from datajunction_server.construction.build import (
     build_metric_nodes,
     build_node,
     rename_columns,
+    validate_shared_dimensions,
 )
 from datajunction_server.construction.dj_query import build_dj_query
 from datajunction_server.errors import (
@@ -69,7 +70,7 @@ from datajunction_server.sql.parsing import ast
 from datajunction_server.sql.parsing.backends.antlr4 import SqlSyntaxError, parse
 from datajunction_server.sql.parsing.backends.exceptions import DJParseException
 from datajunction_server.typing import END_JOB_STATES, UTCDatetime
-from datajunction_server.utils import LOOKUP_CHARS, SEPARATOR, amenable_name
+from datajunction_server.utils import LOOKUP_CHARS, SEPARATOR
 
 _logger = logging.getLogger(__name__)
 
@@ -719,6 +720,11 @@ def validate_cube(  # pylint: disable=too-many-locals
             message=("Metrics and dimensions must be part of a common catalog"),
         )
 
+    validate_shared_dimensions(
+        metric_nodes,
+        [dim.full_name() for dim in dimensions],
+        [],
+    )
     return metrics, metric_nodes, dimension_nodes, dimensions, catalog
 
 
@@ -900,7 +906,7 @@ def build_sql_for_multiple_metrics(  # pylint: disable=too-many-arguments,too-ma
         access_control=access_control,
     )
     columns = [
-        assemble_column_metadata(col, metrics)  # type: ignore
+        assemble_column_metadata(col)  # type: ignore
         for col in query_ast.select.projection
     ]
     return (
@@ -1273,27 +1279,29 @@ def hard_delete_node(
 
 def assemble_column_metadata(
     column: ast.Column,
-    node_name: Union[List[str], str],
+    # node_name: Union[List[str], str],
 ) -> ColumnMetadata:
     """
     Extract column metadata from AST
     """
-    column_name_on_ast = column.alias_or_name.name
-    if not isinstance(node_name, List):
-        node_name = [node_name]
-    node_name_mapping = {amenable_name(name): name for name in node_name}
-    metric_node = node_name_mapping.get(column_name_on_ast)
-
-    amenable_separator = f"_{LOOKUP_CHARS.get(SEPARATOR)}_"
-    column_name_on_node = column_name_on_ast.split(amenable_separator)[-1]
-    reference_node_name = ".".join(column_name_on_ast.split(amenable_separator)[0:-1])
-    if metric_node:
-        column_name_on_node = column_name_on_ast
-        reference_node_name = metric_node
-
-    return ColumnMetadata(
-        name=column_name_on_ast,
+    metadata = ColumnMetadata(
+        name=column.alias_or_name.name,
         type=str(column.type),
-        column=column_name_on_node,
-        node=reference_node_name,
+        column=(
+            column.semantic_entity.split(SEPARATOR)[-1]
+            if hasattr(column, "semantic_entity") and column.semantic_entity
+            else None
+        ),
+        node=(
+            SEPARATOR.join(column.semantic_entity.split(SEPARATOR)[:-1])
+            if hasattr(column, "semantic_entity") and column.semantic_entity
+            else None
+        ),
+        semantic_entity=column.semantic_entity
+        if hasattr(column, "semantic_entity")
+        else None,
+        semantic_type=column.semantic_type
+        if hasattr(column, "semantic_type")
+        else None,
     )
+    return metadata

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/main.py

@@ -15,6 +15,8 @@ from typing import TYPE_CHECKING
 
 from fastapi import Depends, FastAPI, Request
 from fastapi.responses import JSONResponse, RedirectResponse, Response
+from fastapi_cache import FastAPICache
+from fastapi_cache.backends.inmemory import InMemoryBackend
 from jose import JWTError
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from sqlmodel import select
@@ -42,6 +44,7 @@ from datajunction_server.api import (
 )
 from datajunction_server.api.access.authentication import whoami
 from datajunction_server.api.attributes import default_attribute_types
+from datajunction_server.api.catalogs import default_catalog
 from datajunction_server.api.graphql.main import graphql_app
 from datajunction_server.constants import AUTH_COOKIE, LOGGED_IN_FLAG_COOKIE
 from datajunction_server.errors import DJException
@@ -64,7 +67,7 @@ config.fileConfig(
     disable_existing_loggers=False,
 )
 
-dependencies = [Depends(default_attribute_types)]
+dependencies = [Depends(default_attribute_types), Depends(default_catalog)]
 
 app = FastAPI(
     title=settings.name,
@@ -105,6 +108,14 @@ app.include_router(graphql_app, prefix="/graphql")
 app.include_router(whoami.router)
 
 
+@app.on_event("startup")
+async def startup():
+    """
+    Initialize FastAPI cache when the server starts up
+    """
+    FastAPICache.init(InMemoryBackend(), prefix="inmemory-cache")
+
+
 @app.exception_handler(DJException)
 async def dj_exception_handler(  # pylint: disable=unused-argument
     request: Request,

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/materializations.py

@@ -14,11 +14,13 @@ from sqlmodel import Session
 from datajunction_server.api.helpers import get_node_by_name
 from datajunction_server.errors import DJDoesNotExistException, DJException
 from datajunction_server.internal.access.authentication.http import SecureAPIRouter
+from datajunction_server.internal.access.authorization import validate_access
 from datajunction_server.internal.materializations import (
     create_new_materialization,
     schedule_materialization_jobs,
 )
 from datajunction_server.materialization.jobs import MaterializationJob
+from datajunction_server.models import access
 from datajunction_server.models.history import ActivityType, EntityType, History
 from datajunction_server.models.materialization import (
     MaterializationConfigInfoUnified,
@@ -54,6 +56,9 @@ def upsert_materialization(  # pylint: disable=too-many-locals
     session: Session = Depends(get_session),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     current_user: Optional[User] = Depends(get_current_user),
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> JSONResponse:
     """
     Add or update a materialization of the specified node. If a node_name is specified
@@ -69,7 +74,12 @@ def upsert_materialization(  # pylint: disable=too-many-locals
     old_materializations = {mat.name: mat for mat in current_revision.materializations}
 
     # Create a new materialization
-    new_materialization = create_new_materialization(session, current_revision, data)
+    new_materialization = create_new_materialization(
+        session,
+        current_revision,
+        data,
+        validate_access,
+    )
 
     # Check to see if a materialization for this engine already exists with the exact same config
     existing_materialization = old_materializations.get(new_materialization.name)

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/nodes.py

@@ -9,10 +9,12 @@ from typing import List, Optional, Union, cast
 
 from fastapi import BackgroundTasks, Depends, Query, Response
 from fastapi.responses import JSONResponse
+from fastapi_cache.decorator import cache
 from sqlalchemy.sql.operators import is_
 from sqlmodel import Session, select
 from starlette.requests import Request
 
+from datajunction_server.api.catalogs import UNKNOWN_CATALOG_ID
 from datajunction_server.api.helpers import (
     activate_node,
     deactivate_node,
@@ -214,6 +216,7 @@ def list_nodes(
 
 
 @router.get("/nodes/details/", response_model=List[NodeIndexItem])
+@cache(expire=settings.index_cache_expire)
 def list_all_nodes_with_details(
     node_type: Optional[NodeType] = None,
     *,
@@ -351,6 +354,9 @@ def create_source(
     session: Session = Depends(get_session),
     current_user: Optional[User] = Depends(get_current_user),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> NodeOutput:
     """
     Create a source node. If columns are not provided, the source node's schema
@@ -365,6 +371,7 @@ def create_source(
         session=session,
         current_user=current_user,
         query_service_client=query_service_client,
+        validate_access=validate_access,
     ):
         return recreated_node
 
@@ -447,6 +454,9 @@ def create_node(
     current_user: Optional[User] = Depends(get_current_user),
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     background_tasks: BackgroundTasks,
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> NodeOutput:
     """
     Create a node.
@@ -466,6 +476,7 @@ def create_node(
         current_user=current_user,
         query_service_client=query_service_client,
         background_tasks=background_tasks,
+        validate_access=validate_access,
     ):
         return recreated_node  # pragma: no cover
 
@@ -528,6 +539,9 @@ def create_cube(
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     current_user: Optional[User] = Depends(get_current_user),
     background_tasks: BackgroundTasks,
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> NodeOutput:
     """
     Create a cube node.
@@ -542,6 +556,7 @@ def create_cube(
         current_user=current_user,
         query_service_client=query_service_client,
         background_tasks=background_tasks,
+        validate_access=validate_access,
     ):
         return recreated_node  # pragma: no cover
 
@@ -640,7 +655,8 @@ def link_dimension(  # pylint: disable=too-many-arguments
         node_type=NodeType.DIMENSION,
     )
     if (
-        dimension_node.current.catalog is not None
+        dimension_node.current.catalog_id != UNKNOWN_CATALOG_ID
+        and dimension_node.current.catalog is not None
         and node.current.catalog.name != dimension_node.current.catalog.name
     ):
         raise DJException(
@@ -909,6 +925,9 @@ def update_node(
     query_service_client: QueryServiceClient = Depends(get_query_service_client),
     current_user: Optional[User] = Depends(get_current_user),
     background_tasks: BackgroundTasks,
+    validate_access: access.ValidateAccessFn = Depends(  # pylint: disable=W0621
+        validate_access,
+    ),
 ) -> NodeOutput:
     """
     Update a node.
@@ -920,6 +939,7 @@ def update_node(
         query_service_client=query_service_client,
         current_user=current_user,
         background_tasks=background_tasks,
+        validate_access=validate_access,
     )
     return node  # type: ignore
 

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/api/sql.py

@@ -103,7 +103,7 @@ def get_sql(
         access_control=access_control,
     )
     columns = [
-        assemble_column_metadata(col, node_name)  # type: ignore
+        assemble_column_metadata(col)  # type: ignore
         for col in query_ast.select.projection
     ]
     return TranslatedSQL(

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/config.py

@@ -88,6 +88,9 @@ class Settings(
     # Google OAuth application client secret file
     google_oauth_client_secret_file: Optional[str] = None
 
+    # Interval in seconds with which to expire caching of any indexes
+    index_cache_expire = 60
+
     @property
     def celery(self) -> Celery:
         """

{datajunction_server-0.0.1a35 → datajunction_server-0.0.1a35.dev1}/datajunction_server/constants.py

@@ -20,4 +20,4 @@ AUTH_COOKIE = "__dj"
 LOGGED_IN_FLAG_COOKIE = "__djlif"
 
 # Maximum amount of nodes to return for requests to list all nodes
-NODE_LIST_MAX = 50000
+NODE_LIST_MAX = 1000