datajunction-reflection 0.0.1a1__py3-none-any.whl

datajunction_reflection-0.0.1a1.dist-info/METADATA

@@ -0,0 +1,35 @@
+Metadata-Version: 2.1
+Name: datajunction-reflection
+Version: 0.0.1a1
+Summary: OSS Implementation of a DataJunction Reflection Service
+Project-URL: repository, https://github.com/DataJunction/dj
+Author-email: DataJunction Authors <roberto@dealmeida.net>
+License: MIT
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: <4.0,>=3.8
+Requires-Dist: celery[redis]>=5.2.3
+Requires-Dist: djopenapi
+Requires-Dist: importlib-metadata
+Requires-Dist: pydantic
+Requires-Dist: python-dotenv==0.19.2
+Requires-Dist: requests>=2.26.0
+Provides-Extra: uvicorn
+Requires-Dist: uvicorn[standard]>=0.21.1; extra == 'uvicorn'
+Description-Content-Type: text/markdown
+
+# DJ Reflection Service
+
+The reflection service polls the DJ core service for all nodes with associated tables, whether source
+tables or materialized tables. For each node, it refreshes the node's schema based on the associated
+table's schema that it retrieves from the query service. It also retrieves the available partitions and
+the valid through timestamp of these tables and reflects them accordingly to DJ core.
+
+This service uses a celery beat scheduler, with a configurable polling interval that defaults to once per
+hour and async tasks for each node's reflection.

datajunction_reflection-0.0.1a1.dist-info/RECORD

@@ -0,0 +1,11 @@
+djrs/__about__.py,sha256=mlybO8bzNO9GojXJH7D8hqtzoQxcupTM5lXnSjq6_f8,50
+djrs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+djrs/config.py,sha256=ImLCyAULcoNFmKSmNXuGlP6tw4J6AHKI4ultb6qGiho,613
+djrs/worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+djrs/worker/app.py,sha256=XdaZshykwg2CypN10XefPhBPaYSRkzdXeB1cRzQpjDk,174
+djrs/worker/tasks.py,sha256=MyKita0jeJpL1h6lNJYOQzUZzht1azw4n72ZGSKmgsg,3871
+djrs/worker/utils.py,sha256=VEfVEPmrGc86j2GY7Bal5NTQQSAfiuszf90xf-vDLZ8,1276
+datajunction_reflection-0.0.1a1.dist-info/METADATA,sha256=nC-K4XdFR29AWRK9ywsUCI7LKdo6h6nq4ZUsPD7-CgU,1555
+datajunction_reflection-0.0.1a1.dist-info/WHEEL,sha256=KGYbc1zXlYddvwxnNty23BeaKzh7YuoSIvIMO4jEhvw,87
+datajunction_reflection-0.0.1a1.dist-info/licenses/LICENSE,sha256=Q5EUOoPvfa4k1Az3B6y-bddOppd_h7sa30WgFDZOxsU,1059
+datajunction_reflection-0.0.1a1.dist-info/RECORD,,

datajunction_reflection-0.0.1a1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.17.1
+Root-Is-Purelib: true
+Tag: py3-none-any

datajunction_reflection-0.0.1a1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DJ
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

djrs/__about__.py

@@ -0,0 +1,4 @@
+"""
+Version for Hatch
+"""
+__version__ = "0.0.1a1"

djrs/config.py

@@ -0,0 +1,26 @@
+"""Reflection service settings."""
+from functools import lru_cache
+
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    """
+    Default settings for the reflection service.
+    """
+
+    core_service: str = "http://dj:8000"
+    query_service: str = "http://djqs:8001"
+    celery_broker: str = "redis://djrs-redis:6379/1"
+    celery_results_backend: str = "redis://djrs-redis:6379/2"
+
+    # Set the number of seconds to wait in between polling
+    polling_interval: int = 3600
+
+
+@lru_cache
+def get_settings() -> Settings:
+    """
+    Return a cached settings object.
+    """
+    return Settings()

djrs/worker/app.py

@@ -0,0 +1,7 @@
+"""
+Celery app that does the polling of nodes in DJ and then subsequent
+queueing of reflection tasks.
+"""
+from djrs.worker.utils import get_celery
+
+celery_app = get_celery()

djrs/worker/tasks.py

@@ -0,0 +1,133 @@
+"""Reflection service celery tasks."""
+import datetime
+import json
+from abc import ABC
+from typing import Dict
+
+import celery
+import requests
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from djopenapi.model.availability_state_base import AvailabilityStateBase
+from djopenapi.model.update_node import UpdateNode
+
+from djrs.worker.app import celery_app
+from djrs.worker.utils import get_dj_client, get_settings
+
+logger = get_task_logger(__name__)
+
+
+class ReflectionServiceTask(celery.Task, ABC):
+    """
+    Base reflection service task.
+    """
+
+    abstract = True
+
+    def on_failure(self, exc, task_id, *args, **kwargs):
+        logger.exception("%s failed: %s", task_id, exc)
+
+
+@shared_task(queue="celery", name="djrs.worker.app.refresh", base=ReflectionServiceTask)
+def refresh():
+    """
+    Find available DJ nodes and kick off reflection tasks for
+    nodes with associated tables.
+    """
+    dj_api = get_dj_client()
+
+    catalogs_to_engines = {
+        catalog["name"]: catalog["engines"][0] if catalog["engines"] else None
+        for catalog in json.loads(
+            dj_api.list_catalogs_catalogs_get(skip_deserialization=True).response.data,
+        )
+    }
+
+    all_nodes = {
+        node["name"]: node
+        for node in json.loads(
+            dj_api.read_nodes_nodes_get(skip_deserialization=True).response.data,
+        )
+    }
+
+    tasks = []
+    for node_name, node in all_nodes.items():
+        if node["catalog"] and node["schema_"] and node["table"]:
+            task = celery_app.send_task(
+                "djrs.worker.tasks.reflect",
+                (
+                    node_name,
+                    node["catalog"]["name"],
+                    node["schema_"],
+                    node["table"],
+                    catalogs_to_engines[node["catalog"]["name"]],
+                ),
+            )
+            tasks.append(task)
+
+
+@shared_task(
+    queue="celery",
+    name="djrs.worker.tasks.reflect",
+    base=ReflectionServiceTask,
+)
+def reflect(
+    node_name: str,
+    catalog: str,
+    schema: str,
+    table: str,
+    engine: Dict[str, str],
+):
+    """
+    This reflects the state of the node's associated table, whether
+    external or materialized, back to the DJ core service.
+    """
+    logger.info(f"Reflecting node={node_name}, table={table} to DJ core")
+    settings = get_settings()
+    dj_api = get_dj_client()
+
+    # Update table columns
+    response = requests.get(
+        f"{settings.query_service}/table/{catalog}.{schema}.{table}/columns/",
+        params={
+            "engine": engine["name"],
+            "engine_version": engine["version"],
+        },
+        timeout=30,
+    )
+    table_columns = response.json()["columns"]
+    update_columns_response = dj_api.update_node_nodes_name_patch(
+        body=UpdateNode(
+            columns={col["name"]: {"type": col["type"]} for col in table_columns},
+        ),
+        path_params={"name": node_name},
+        skip_deserialization=True,
+    ).response
+
+    logger.info(
+        f"Update node columns for `{node_name}` response: "
+        f"{update_columns_response.reason}",
+    )
+
+    # pylint: disable=fixme
+    # TODO: Post actual availability state when information available
+    availability_state_response = (
+        dj_api.add_availability_data_node_name_availability_post(
+            body=AvailabilityStateBase(
+                catalog=catalog,
+                max_partition=[],
+                min_partition=[],
+                table=table,
+                valid_through_ts=int(datetime.datetime.now().timestamp()),
+                schema_=schema,
+            ),
+            path_params={"node_name": node_name},
+            skip_deserialization=True,
+        ).response
+    )
+
+    logger.info(
+        "Post availability state for `%s` response: %s",
+        node_name,
+        availability_state_response.reason,
+    )

djrs/worker/utils.py

@@ -0,0 +1,49 @@
+"""Utility functions for retrieving API clients."""
+import os
+from typing import Optional
+
+import djopenapi
+from celery import Celery
+from djopenapi.apis.tags import default_api
+
+from djrs.config import get_settings
+
+
+def get_dj_client(
+    core_service: Optional[str] = None,
+) -> default_api.DefaultApi:
+    """
+    Return DJ API client
+    """
+    settings = get_settings()
+    configuration = djopenapi.Configuration(
+        host=core_service or settings.core_service,
+    )
+    api_client = djopenapi.ApiClient(configuration)
+    return default_api.DefaultApi(api_client)
+
+
+def get_celery() -> Celery:
+    """
+    core celery app
+    """
+
+    settings = get_settings()
+
+    celery_app = Celery(__name__, include=["djrs.worker.app", "djrs.worker.tasks"])
+    celery_app.conf.broker_url = os.environ.get(
+        "CELERY_BROKER_URL",
+        settings.celery_broker,
+    )
+    celery_app.conf.result_backend = os.environ.get(
+        "CELERY_RESULT_BACKEND",
+        settings.celery_results_backend,
+    )
+    celery_app.conf.imports = ["djrs.worker.app", "djrs.worker.tasks"]
+    celery_app.conf.beat_schedule = {
+        "refresh": {
+            "task": "djrs.worker.app.refresh",
+            "schedule": settings.polling_interval,
+        },
+    }
+    return celery_app