datajunction-query 0.0.1a1.dev0__py3-none-any.whl → 0.0.1a31__py3-none-any.whl

alembic/versions/2023_10_09_1858-f3407a1ec625_add_type_and_extra_para_ms_field_for_.py

@@ -0,0 +1,36 @@
+"""Add type and extra_params field for engines
+
+Revision ID: f3407a1ec625
+Revises: a7e11a2438b4
+Create Date: 2023-10-09 18:58:57.462134+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+import sqlmodel
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "f3407a1ec625"
+down_revision = "a7e11a2438b4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("engine", sa.Column("extra_params", sa.JSON(), nullable=True))
+    op.add_column(
+        "engine",
+        sa.Column("type", sa.Enum("DUCKDB", name="enginetype"), nullable=False),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("engine", "type")
+    op.drop_column("engine", "extra_params")
+    # ### end Alembic commands ###

{datajunction_query-0.0.1a1.dev0.dist-info → datajunction_query-0.0.1a31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datajunction-query
-Version: 0.0.1a1.dev0
+Version: 0.0.1a31
 Summary: OSS Implementation of a DataJunction Query Service
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <roberto@dealmeida.net>
@@ -30,6 +30,7 @@ Requires-Dist: sqlalchemy-utils>=0.37.7
 Requires-Dist: sqlalchemy<2.0.0,>=1.4.41
 Requires-Dist: sqlmodel<1.0.0,>=0.0.8
 Requires-Dist: sqlparse<1.0.0,>=0.4.3
+Requires-Dist: toml>=0.10.2
 Provides-Extra: uvicorn
 Requires-Dist: uvicorn[standard]>=0.21.1; extra == 'uvicorn'
 Description-Content-Type: text/x-rst

{datajunction_query-0.0.1a1.dev0.dist-info → datajunction_query-0.0.1a31.dist-info}/RECORD

@@ -1,31 +1,32 @@
 alembic/env.py,sha256=cUXPBnE3hXpjINWb4vD8TKE7w3UnkYOjwT9DLa7Y5lg,2207
 alembic/versions/2023_02_28_0541-a7e11a2438b4_initial_migration.py,sha256=DRkvqqkbe8xhj34Y09NGqCv7Ohn73Ez2IuMEMNqgu1A,3146
-djqs/__about__.py,sha256=SOGstYf7wRCzgPecz9Cx_cAd5vqNX_oDj1M-ijb7o2w,55
+alembic/versions/2023_10_09_1858-f3407a1ec625_add_type_and_extra_para_ms_field_for_.py,sha256=HBdeQDzSALCGxUl6p7qSBhjWwtwIoh4QukGRNydg7fw,979
+djqs/__about__.py,sha256=dXBG1UdX2tVdiwoix_NgzSIcJ5HnXKM_3_Ajtqtuvzo,51
 djqs/__init__.py,sha256=nN5-uJoSVEwuc8n-wMygqeF0Xhxi_zqqbCgutZvAt3E,384
-djqs/config.py,sha256=5-q1r2OVb3Bwe3RbzXQ58CW6phEIuEhiHj-uFeR2ocM,1075
+djqs/config.py,sha256=Flva9W8McnTunUlHGhYd0CsgbsjWbZIoPW25ZTzFxws,3029
 djqs/constants.py,sha256=T-sO5C5-NNdEywGglTIu6bjT1VooFwW4XWNA_ZwSYM8,412
-djqs/engine.py,sha256=TGp3mqPreSWqQAJ3F_BUkDmlXUC-KrKCL_LT3av7K-0,6287
+djqs/engine.py,sha256=EJ-Z0guqA20H6b1U5_Of3fwvHJ_AknOrxY_ewMXqRFk,5114
 djqs/exceptions.py,sha256=oHVFngJJGIPmZL7-AwpW9UnP0FDcncbjPcWJpv1SXAU,4888
 djqs/fixes.py,sha256=TcXnh0I1z4vEAupPatzrvnqyToGihndnxmLJtIn-_Z8,33
 djqs/typing.py,sha256=ByasjhXZza2VeRCj1mMhYeCTuW_xXc5P6OblyLYqtu4,6217
 djqs/utils.py,sha256=R-t2A8RvwLu2Lxcs8r5kxQ7AfuDRKJ7G9XcjfdXAhvQ,2075
 djqs/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-djqs/api/catalogs.py,sha256=W4G4yDRwBy3w82ZBJ4QVxZVuC1At4iSN8ZOlCWu_GfU,2948
-djqs/api/engines.py,sha256=LvrOwpZXwirdqtKkyOCxRe6d-eeNPxv2W-I7owSKF3Y,1517
-djqs/api/helpers.py,sha256=bzXPuTuHzAAxHy-pwvKjcL61wMuuoOMZc81rBbqu5po,2651
-djqs/api/main.py,sha256=Z4ICNFInqOX5w9jXUU3P4yPXwWa9IH6IEPszG3Vl_0Q,1390
+djqs/api/catalogs.py,sha256=I_i8-Nok0V3ND1FpOCQ5Nmgaof5MLH8i0oeTjvDxQgw,3062
+djqs/api/engines.py,sha256=278fsgIgQXe4WAmbfbxoA08ALe57us_zKlaK_hbiDrc,1611
+djqs/api/helpers.py,sha256=z-e_fFPE_dSvFax8wKbssS7VQvxo1E5V915zc8WFYSY,2402
+djqs/api/main.py,sha256=D99c_I2sZm8vXFuW2ZKqV_NqfahJ1mSvytPShVDlsi8,1728
 djqs/api/queries.py,sha256=NAZd2T9_CgWYKsxH7RxTUj6SIA2t8C4HEZHl_n47LL8,5485
 djqs/api/tables.py,sha256=5qADafYVkOBaPhq1s9s3Tf_RuTE1Eiqnnvi5bUYB7Io,1457
 djqs/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 djqs/models/catalog.py,sha256=o3Wlghwr6ax4Eo-3zE5Mdq478xyUxeaf2o3whFprXtU,2038
-djqs/models/engine.py,sha256=SwikYBCCD9GvFhqsROoAz3G8PaxrxdKR8VQ7m2i0g2w,510
+djqs/models/engine.py,sha256=LcklXgew8O5wFELhI24p2HgOXcMPYrU9uGJ6PIQxdxM,861
 djqs/models/query.py,sha256=FeOOhVGAU8lOajcVYrd2GsOz_XcSWHDy8-t74mGKhhw,3332
 djqs/models/table.py,sha256=7HmiXWHBWEthqdwPYLlsOgeKD-w9urCHoC4xaQ1RnM8,238
 docker/duckdb_load.py,sha256=xOX5fCsJkrWa_7kYF6FEqkK8GWn2S1CmjvRCRIpLmxc,229
 docker/spark_load_roads.py,sha256=PN5D0eZw1MrXZ4Cax4aZGBWyRMwHVVVJBlugraQhiu8,574
 scripts/generate-openapi.py,sha256=m2P1EEFbF6glwmZj6ywoteo8OLvT-7b0IaooWQWqwU4,622
-datajunction_query-0.0.1a1.dev0.dist-info/METADATA,sha256=C1K97haBfvnfyGo024TUzx1sqM_oxOuXDu93ZIR1f8M,9393
-datajunction_query-0.0.1a1.dev0.dist-info/WHEEL,sha256=9QBuHhg6FNW7lppboF2vKVbCGTVzsFykgRQjjlajrhA,87
-datajunction_query-0.0.1a1.dev0.dist-info/licenses/AUTHORS.rst,sha256=G9YmXPfQ0kAuxqlCwvWVvUnZitP9hAc-rPfZ5q7Pv1U,235
-datajunction_query-0.0.1a1.dev0.dist-info/licenses/LICENSE.txt,sha256=KuSxhVgPuUGoYWphJig4POcTAIUNLUj8vOx-cqQFMj8,1081
-datajunction_query-0.0.1a1.dev0.dist-info/RECORD,,
+datajunction_query-0.0.1a31.dist-info/METADATA,sha256=E8rWMok6h5XPxFCddzKcuofFMsshoxN2l9NSH1vWUXE,9417
+datajunction_query-0.0.1a31.dist-info/WHEEL,sha256=9QBuHhg6FNW7lppboF2vKVbCGTVzsFykgRQjjlajrhA,87
+datajunction_query-0.0.1a31.dist-info/licenses/AUTHORS.rst,sha256=G9YmXPfQ0kAuxqlCwvWVvUnZitP9hAc-rPfZ5q7Pv1U,235
+datajunction_query-0.0.1a31.dist-info/licenses/LICENSE.txt,sha256=KuSxhVgPuUGoYWphJig4POcTAIUNLUj8vOx-cqQFMj8,1081
+datajunction_query-0.0.1a31.dist-info/RECORD,,

djqs/__about__.py

@@ -1,4 +1,4 @@
 """
 Version for Hatch
 """
-__version__ = "0.0.1a1.dev0"
+__version__ = "0.0.1a31"

djqs/api/catalogs.py

@@ -17,10 +17,11 @@ from djqs.models.engine import BaseEngineInfo
 from djqs.utils import get_session
 
 _logger = logging.getLogger(__name__)
-router = APIRouter(tags=["Catalogs & Engines"])
+get_router = APIRouter(tags=["Catalogs & Engines"])
+post_router = APIRouter(tags=["Catalogs & Engines - Dynamic Configuration"])
 
 
-@router.get("/catalogs/", response_model=List[CatalogInfo])
+@get_router.get("/catalogs/", response_model=List[CatalogInfo])
 def list_catalogs(*, session: Session = Depends(get_session)) -> List[CatalogInfo]:
     """
     List all available catalogs
@@ -28,7 +29,7 @@ def list_catalogs(*, session: Session = Depends(get_session)) -> List[CatalogInf
     return list(session.exec(select(Catalog)))
 
 
-@router.get("/catalogs/{name}/", response_model=CatalogInfo)
+@get_router.get("/catalogs/{name}/", response_model=CatalogInfo)
 def read_catalog(name: str, *, session: Session = Depends(get_session)) -> CatalogInfo:
     """
     Return a catalog by name
@@ -36,7 +37,7 @@ def read_catalog(name: str, *, session: Session = Depends(get_session)) -> Catal
     return get_catalog(session, name)
 
 
-@router.post("/catalogs/", response_model=CatalogInfo, status_code=201)
+@post_router.post("/catalogs/", response_model=CatalogInfo, status_code=201)
 def add_catalog(
     data: CatalogInfo,
     *,
@@ -70,7 +71,11 @@ def add_catalog(
     return catalog
 
 
-@router.post("/catalogs/{name}/engines/", response_model=CatalogInfo, status_code=201)
+@post_router.post(
+    "/catalogs/{name}/engines/",
+    response_model=CatalogInfo,
+    status_code=201,
+)
 def add_engines_to_catalog(
     name: str,
     data: List[BaseEngineInfo],

djqs/api/engines.py

@@ -12,10 +12,11 @@ from djqs.api.helpers import get_engine
 from djqs.models.engine import BaseEngineInfo, Engine, EngineInfo
 from djqs.utils import get_session
 
-router = APIRouter(tags=["Catalogs & Engines"])
+get_router = APIRouter(tags=["Catalogs & Engines"])
+post_router = APIRouter(tags=["Catalogs & Engines - Dynamic Configuration"])
 
 
-@router.get("/engines/", response_model=List[EngineInfo])
+@get_router.get("/engines/", response_model=List[EngineInfo])
 def list_engines(*, session: Session = Depends(get_session)) -> List[EngineInfo]:
     """
     List all available engines
@@ -23,7 +24,7 @@ def list_engines(*, session: Session = Depends(get_session)) -> List[EngineInfo]
     return list(session.exec(select(Engine)))
 
 
-@router.get("/engines/{name}/{version}/", response_model=BaseEngineInfo)
+@get_router.get("/engines/{name}/{version}/", response_model=BaseEngineInfo)
 def list_engine(
     name: str, version: str, *, session: Session = Depends(get_session)
 ) -> BaseEngineInfo:
@@ -33,7 +34,7 @@ def list_engine(
     return get_engine(session, name, version)
 
 
-@router.post("/engines/", response_model=BaseEngineInfo, status_code=201)
+@post_router.post("/engines/", response_model=BaseEngineInfo, status_code=201)
 def add_engine(
     data: EngineInfo,
     *,

djqs/api/helpers.py

@@ -9,7 +9,6 @@ from sqlalchemy import inspect
 from sqlalchemy.exc import NoResultFound, NoSuchTableError, OperationalError
 from sqlmodel import Session, create_engine, select
 
-from djqs.engine import describe_table_via_spark, get_spark_session
 from djqs.exceptions import DJException
 from djqs.models.catalog import Catalog
 from djqs.models.engine import Engine
@@ -59,14 +58,6 @@ def get_columns(
     if not uri:
         raise DJException("Cannot retrieve columns without a uri")
 
-    if uri.startswith("spark://"):
-        spark = get_spark_session()
-        return describe_table_via_spark(
-            spark,
-            schema,
-            table,
-        )
-
     engine = create_engine(uri, **extra_params)
     try:
         inspector = inspect(engine)

djqs/api/main.py

@@ -5,7 +5,7 @@ Main DJ query server app.
 # All the models need to be imported here so that SQLModel can define their
 # relationships at runtime without causing circular imports.
 # See https://sqlmodel.tiangolo.com/tutorial/code-structure/#make-circular-imports-work.
-# pylint: disable=unused-import
+# pylint: disable=unused-import,expression-not-assigned
 
 import logging
 
@@ -14,13 +14,16 @@ from fastapi.responses import JSONResponse
 
 from djqs import __version__
 from djqs.api import catalogs, engines, queries, tables
+from djqs.config import load_djqs_config
 from djqs.exceptions import DJException
-from djqs.utils import get_settings
+from djqs.utils import get_session, get_settings
 
 _logger = logging.getLogger(__name__)
 
-
 settings = get_settings()
+session = next(get_session())
+load_djqs_config(settings=settings, session=session)
+
 app = FastAPI(
     title=settings.name,
     description=settings.description,
@@ -30,10 +33,12 @@ app = FastAPI(
         "url": "https://mit-license.org/",
     },
 )
-app.include_router(catalogs.router)
-app.include_router(engines.router)
+app.include_router(catalogs.get_router)
+app.include_router(engines.get_router)
 app.include_router(queries.router)
 app.include_router(tables.router)
+app.include_router(catalogs.post_router) if settings.enable_dynamic_config else None
+app.include_router(engines.post_router) if settings.enable_dynamic_config else None
 
 
 @app.exception_handler(DJException)

djqs/config.py

@@ -2,11 +2,20 @@
 Configuration for the query service
 """
 
+import json
 from datetime import timedelta
+from typing import Optional
 
+import toml
+import yaml
 from cachelib.base import BaseCache
 from cachelib.file import FileSystemCache
 from pydantic import BaseSettings
+from sqlmodel import Session, delete, select
+
+from djqs.exceptions import DJException
+from djqs.models.catalog import Catalog, CatalogEngines
+from djqs.models.engine import Engine
 
 
 class Settings(BaseSettings):  # pylint: disable=too-few-public-methods
@@ -34,3 +43,54 @@ class Settings(BaseSettings):  # pylint: disable=too-few-public-methods
 
     # How long to wait when pinging databases to find out the fastest online database.
     do_ping_timeout: timedelta = timedelta(seconds=5)
+
+    # Configuration file for catalogs and engines
+    configuration_file: Optional[str] = None
+
+    # Enable setting catalog and engine config via REST API calls
+    enable_dynamic_config: bool = True
+
+
+def load_djqs_config(settings: Settings, session: Session) -> None:  # pragma: no cover
+    """
+    Load the DJQS config file into the server metadata database
+    """
+    config_file = settings.configuration_file if settings.configuration_file else None
+    if not config_file:
+        return
+
+    session.exec(delete(Catalog))
+    session.exec(delete(Engine))
+    session.exec(delete(CatalogEngines))
+    session.commit()
+
+    with open(config_file, mode="r", encoding="utf-8") as filestream:
+
+        def unknown_filetype():
+            raise DJException(message=f"Unknown config file type: {config_file}")
+
+        data = (
+            yaml.safe_load(filestream)
+            if any([config_file.endswith(".yml"), config_file.endswith(".yaml")])
+            else toml.load(filestream)
+            if config_file.endswith(".toml")
+            else json.load(filestream)
+            if config_file.endswith(".json")
+            else unknown_filetype()
+        )
+
+    for engine in data["engines"]:
+        session.add(Engine.parse_obj(engine))
+    session.commit()
+
+    for catalog in data["catalogs"]:
+        attached_engines = []
+        catalog_engines = catalog.pop("engines")
+        for name in catalog_engines:
+            attached_engines.append(
+                session.exec(select(Engine).where(Engine.name == name)).one(),
+            )
+        catalog_entry = Catalog.parse_obj(catalog)
+        catalog_entry.engines = attached_engines
+        session.add(catalog_entry)
+    session.commit()

djqs/engine.py

@@ -4,17 +4,16 @@ Query related functions.
 
 import logging
 from datetime import datetime, timezone
-from typing import List, Optional, Tuple
+from typing import List, Tuple
 
 import duckdb
 import sqlparse
-from pyspark.sql import SparkSession  # pylint: disable=import-error
 from sqlalchemy import create_engine, text
 from sqlmodel import Session, select
 
 from djqs.config import Settings
 from djqs.models.catalog import Catalog
-from djqs.models.engine import Engine
+from djqs.models.engine import Engine, EngineType
 from djqs.models.query import (
     ColumnMetadata,
     Query,
@@ -85,11 +84,15 @@ def run_query(
         .where(Engine.name == query.engine_name)
         .where(Engine.version == query.engine_version),
     ).one()
-    if engine.uri == "spark://local[*]":
-        spark = get_spark_session()
-        return run_spark_query(query, spark)
-    if engine.uri == "duckdb://local[*]":
-        conn = duckdb.connect(database="/code/docker/default.duckdb", read_only=False)
+    if engine.type == EngineType.DUCKDB:
+        conn = (
+            duckdb.connect()
+            if engine.uri == "duckdb:///:memory:"
+            else duckdb.connect(
+                database=engine.extra_params["location"],
+                read_only=True,
+            )
+        )
         return run_duckdb_query(query, conn)
     sqla_engine = create_engine(engine.uri, **catalog.extra_params)
     connection = sqla_engine.connect()
@@ -111,49 +114,6 @@ def run_query(
     return output
 
 
-def get_spark_session():
-    """
-    Get a spark session
-    """
-    SparkSession._instantiatedContext = None  # pylint: disable=protected-access
-    spark = (
-        SparkSession.builder.master("local[*]")
-        .appName("djqs")
-        .enableHiveSupport()
-        .getOrCreate()
-    )
-    return spark
-
-
-def run_spark_query(
-    query: Query,
-    spark: SparkSession,
-) -> List[Tuple[str, List[ColumnMetadata], Stream]]:
-    """
-    Run a spark SQL query against the local warehouse
-    """
-    output: List[Tuple[str, List[ColumnMetadata], Stream]] = []
-    results_df = spark.sql(query.submitted_query)
-    rows = results_df.rdd.map(tuple).collect()
-    columns: List[ColumnMetadata] = []
-    output.append((query.submitted_query, columns, rows))
-    return output
-
-
-def describe_table_via_spark(
-    spark: SparkSession,
-    schema: Optional[str],
-    table: str,
-):
-    """
-    Gets the column schemas.
-    """
-    schema_ = f"{schema}." if schema else ""
-    schema_df = spark.sql(f"DESCRIBE TABLE {schema_}{table};")
-    rows = schema_df.rdd.map(tuple).collect()
-    return [{"name": row[0], "type": row[1]} for row in rows]
-
-
 def run_duckdb_query(
     query: Query,
     conn: duckdb.DuckDBPyConnection,

djqs/models/engine.py

@@ -1,10 +1,20 @@
 """
 Models for columns.
 """
+from enum import Enum
+from typing import Dict, Optional
 
-from typing import Optional
+from sqlalchemy.sql.schema import Column as SqlaColumn
+from sqlmodel import JSON, Field, SQLModel
 
-from sqlmodel import Field, SQLModel
+
+class EngineType(Enum):
+    """
+    Supported engine types
+    """
+
+    DUCKDB = "duckdb"
+    SQLALCHEMY = "sqlalchemy"
 
 
 class Engine(SQLModel, table=True):  # type: ignore
@@ -14,8 +24,10 @@ class Engine(SQLModel, table=True):  # type: ignore
 
     id: Optional[int] = Field(default=None, primary_key=True)
     name: str
+    type: EngineType
     version: str
     uri: Optional[str]
+    extra_params: Dict = Field(default={}, sa_column=SqlaColumn(JSON))
 
 
 class BaseEngineInfo(SQLModel):
@@ -25,6 +37,8 @@ class BaseEngineInfo(SQLModel):
 
     name: str
     version: str
+    type: EngineType
+    extra_params: Dict = {}
 
 
 class EngineInfo(BaseEngineInfo):