dbt-copilot-python 0.1.4__tar.gz → 0.1.5.dev2__tar.gz

{dbt_copilot_python-0.1.4 → dbt_copilot_python-0.1.5.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-copilot-python
-Version: 0.1.4
+Version: 0.1.5.dev2
 Summary: Helper functions to run Django and Flask applications in AWS Copilot/ECS.
 Author: Department for Business and Trade Platform Team
 Author-email: sre-team@digital.trade.gov.uk
@@ -9,11 +9,16 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: celery (>=5.3.6,<6.0.0)
 Requires-Dist: opentelemetry-distro (==0.43b0)
 Requires-Dist: opentelemetry-exporter-otlp (==1.22.0)
 Requires-Dist: opentelemetry-instrumentation-wsgi (==0.43b0)
 Requires-Dist: opentelemetry-propagator-aws-xray (>=1.0.1,<2.0.0)
 Requires-Dist: opentelemetry-sdk-extension-aws (>=2.0.1,<3.0.0)
+Requires-Dist: pytest-celery (>=0.0.0,<0.0.1)
+Requires-Dist: pytest-mock (>=3.12.0,<4.0.0)
+Requires-Dist: redis (>=5.0.1,<6.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Description-Content-Type: text/markdown
 
@@ -27,19 +32,17 @@ A set of utility functions for running Django & Flask apps in AWS ECS via AWS Co
 
 ### Installation
 
-```
+```shell
 pip install dbt-copilot-python
 ```
 
 ### Usage
 
-In `settings.py`...
-
-#### ALLOWED_HOSTS
+#### `ALLOWED_HOSTS` setting
 
-Add the ECS container IP to `ALLOWED_HOSTS` so that the Application Load Balancer (ALB) healthcheck will succeed:
+Add the ECS container IP to `ALLOWED_HOSTS` setting so that the Application Load Balancer (ALB) healthcheck will succeed:
 
-```
+```python
 from dbt_copilot_python.network import setup_allowed_hosts
 
 ALLOWED_HOSTS = [...]
@@ -47,7 +50,42 @@ ALLOWED_HOSTS = [...]
 ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS)
 ```
 
-#### DATABASES
+#### Celery health check
+
+Add the health check in your application's Celery config file...
+
+```python
+from dbt_copilot_python.celery_health_check import healthcheck
+
+celery_app = Celery("application_name")
+...
+
+celery_app = healthcheck.setup(celery_app)
+```
+
+Add the health check to the Celery worker service in `docker-compose.yml`...
+
+```yaml
+healthcheck:
+  test: [ "CMD-SHELL", "python -m dbt_copilot_python.celery_health_check.healthcheck" ]
+  interval: 10s
+  timeout: 5s
+  retries: 2
+  start_period: 5s
+```
+
+In your `*-deploy` codebase, add the health check to the Celery worker service in `copilot/celery-worker/manifest.yml`...
+
+```yaml
+healthcheck:
+  command: [ "CMD-SHELL", "launcher bash -c 'python -m dbt_copilot_python.celery_health_check.healthcheck'" ]
+  interval: 10s
+  timeout: 5s
+  retries: 2
+  start_period: 10s
+```
+
+#### `DATABASES` setting
 
 To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secrets Manager, there are two options.
 
@@ -55,7 +93,7 @@ To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secre
 
     Note: This is dependent on the [`dj-database-url`](https://pypi.org/project/dj-database-url/) package which can be installed via `pip install dj-database-url`.
 
-    ```
+    ```python
     import dj_database_url
 
     from dbt_copilot_python.database import database_url_from_env
@@ -69,7 +107,7 @@ To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secre
 
 2. Configure the `DATABASES` setting to use a dictionary containing the settings:
 
-    ```
+    ```python
     from dbt-copilot-python.database import database_from_env
 
     DATABASES = database_from_env("DATABASE_ENV_VAR_KEY")
@@ -83,13 +121,13 @@ To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secre
 
 ### Install dependencies & pre-commit hooks
 
-```
+```shell
 poetry install && poetry run pre-commit install
 ```
 
 ### Run the tests
 
-```
+```shell
 poetry run pytest
 ```
 
@@ -104,7 +142,7 @@ To publish the Python package `dbt-copilot-python`, you will need an API token.
 
 Update the version, as the same version cannot be published to PyPi.
 
-```
+```shell
 poetry version patch
 ```
 
@@ -112,7 +150,7 @@ More options for the `version` command can be found in the [Poetry documentation
 
 Build the Python package.
 
-```
+```shell
 poetry build
 ```
 
@@ -120,7 +158,7 @@ Publish the Python package.
 
 _Note: Make sure your Pull Request (PR) is approved and contains the version upgrade in `pyproject.toml` before publishing the package._
 
-```
+```shell
 poetry publish
 ```
 

{dbt_copilot_python-0.1.4 → dbt_copilot_python-0.1.5.dev2}/README.md

@@ -8,19 +8,17 @@ A set of utility functions for running Django & Flask apps in AWS ECS via AWS Co
 
 ### Installation
 
-```
+```shell
 pip install dbt-copilot-python
 ```
 
 ### Usage
 
-In `settings.py`...
-
-#### ALLOWED_HOSTS
+#### `ALLOWED_HOSTS` setting
 
-Add the ECS container IP to `ALLOWED_HOSTS` so that the Application Load Balancer (ALB) healthcheck will succeed:
+Add the ECS container IP to `ALLOWED_HOSTS` setting so that the Application Load Balancer (ALB) healthcheck will succeed:
 
-```
+```python
 from dbt_copilot_python.network import setup_allowed_hosts
 
 ALLOWED_HOSTS = [...]
@@ -28,7 +26,42 @@ ALLOWED_HOSTS = [...]
 ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS)
 ```
 
-#### DATABASES
+#### Celery health check
+
+Add the health check in your application's Celery config file...
+
+```python
+from dbt_copilot_python.celery_health_check import healthcheck
+
+celery_app = Celery("application_name")
+...
+
+celery_app = healthcheck.setup(celery_app)
+```
+
+Add the health check to the Celery worker service in `docker-compose.yml`...
+
+```yaml
+healthcheck:
+  test: [ "CMD-SHELL", "python -m dbt_copilot_python.celery_health_check.healthcheck" ]
+  interval: 10s
+  timeout: 5s
+  retries: 2
+  start_period: 5s
+```
+
+In your `*-deploy` codebase, add the health check to the Celery worker service in `copilot/celery-worker/manifest.yml`...
+
+```yaml
+healthcheck:
+  command: [ "CMD-SHELL", "launcher bash -c 'python -m dbt_copilot_python.celery_health_check.healthcheck'" ]
+  interval: 10s
+  timeout: 5s
+  retries: 2
+  start_period: 10s
+```
+
+#### `DATABASES` setting
 
 To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secrets Manager, there are two options.
 
@@ -36,7 +69,7 @@ To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secre
 
     Note: This is dependent on the [`dj-database-url`](https://pypi.org/project/dj-database-url/) package which can be installed via `pip install dj-database-url`.
 
-    ```
+    ```python
     import dj_database_url
 
     from dbt_copilot_python.database import database_url_from_env
@@ -50,7 +83,7 @@ To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secre
 
 2. Configure the `DATABASES` setting to use a dictionary containing the settings:
 
-    ```
+    ```python
     from dbt-copilot-python.database import database_from_env
 
     DATABASES = database_from_env("DATABASE_ENV_VAR_KEY")
@@ -64,13 +97,13 @@ To configure the `DATABASES` setting from an RDS JSON object stored in AWS Secre
 
 ### Install dependencies & pre-commit hooks
 
-```
+```shell
 poetry install && poetry run pre-commit install
 ```
 
 ### Run the tests
 
-```
+```shell
 poetry run pytest
 ```
 
@@ -85,7 +118,7 @@ To publish the Python package `dbt-copilot-python`, you will need an API token.
 
 Update the version, as the same version cannot be published to PyPi.
 
-```
+```shell
 poetry version patch
 ```
 
@@ -93,7 +126,7 @@ More options for the `version` command can be found in the [Poetry documentation
 
 Build the Python package.
 
-```
+```shell
 poetry build
 ```
 
@@ -101,7 +134,7 @@ Publish the Python package.
 
 _Note: Make sure your Pull Request (PR) is approved and contains the version upgrade in `pyproject.toml` before publishing the package._
 
-```
+```shell
 poetry publish
 ```
 

dbt_copilot_python-0.1.5.dev2/dbt_copilot_python/celery_health_check/const.py

@@ -0,0 +1,5 @@
+import tempfile
+from pathlib import Path
+
+READINESS_FILE = Path(f"{tempfile.gettempdir()}/celery_ready")
+HEARTBEAT_FILE = Path(f"{tempfile.gettempdir()}/celery_worker_heartbeat")

dbt_copilot_python-0.1.5.dev2/dbt_copilot_python/celery_health_check/healthcheck.py

@@ -0,0 +1,58 @@
+# See https://medium.com/ambient-innovation/health-checks-for-celery-in-kubernetes-cf3274a3e106
+
+import sys
+from datetime import UTC
+from datetime import datetime
+
+from celery import signals
+
+from .const import HEARTBEAT_FILE
+from .const import READINESS_FILE
+from .heartbeat import HeartBeat
+
+
+def on_worker_ready(**_):
+    READINESS_FILE.touch()
+
+
+def on_worker_shutdown(**_):
+    READINESS_FILE.unlink(missing_ok=True)
+
+
+def setup(celery_app=None):
+    signals.worker_ready.connect(on_worker_ready)
+
+    signals.worker_shutdown.connect(on_worker_shutdown)
+
+    celery_app.steps["worker"].add(HeartBeat)
+    return celery_app
+
+
+def check_health():
+    if not READINESS_FILE.is_file():
+        print("Healthcheck: Celery readiness file NOT found.")
+        sys.exit(1)
+
+    if not HEARTBEAT_FILE.is_file():
+        print("Healthcheck: Celery heartbeat file NOT found.")
+        sys.exit(1)
+
+    heartbeat_timestamp = float(HEARTBEAT_FILE.read_text())
+    current_timestamp = datetime.now(UTC).timestamp()
+    time_diff = current_timestamp - heartbeat_timestamp
+    if time_diff > 60:
+        print(
+            "Healthcheck: Celery Worker heartbeat file timestamp "
+            + "DOES NOT match the given constraint."
+        )
+        sys.exit(1)
+
+    print(
+        "Healthcheck: Celery Worker heartbeat file found and timestamp "
+        + "matches the given constraint."
+    )
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    check_health()

dbt_copilot_python-0.1.5.dev2/dbt_copilot_python/celery_health_check/heartbeat.py

@@ -0,0 +1,29 @@
+from datetime import UTC
+from datetime import datetime
+
+from celery import bootsteps
+
+from .const import HEARTBEAT_FILE
+
+
+class HeartBeat(bootsteps.StartStopStep):
+    requires = {"celery.worker.components:Timer"}
+
+    def __init__(self, parent, **kwargs):
+        super().__init__(parent, **kwargs)
+        self.requests = []
+        self.tref = None
+
+    def start(self, worker):
+        self.tref = worker.timer.call_repeatedly(
+            1.0,
+            self.update_heartbeat_file,
+            (worker,),
+            priority=10,
+        )
+
+    def stop(self, worker):
+        HEARTBEAT_FILE.unlink(missing_ok=True)
+
+    def update_heartbeat_file(self, worker):
+        HEARTBEAT_FILE.write_text(str(datetime.timestamp(datetime.now(UTC))))

{dbt_copilot_python-0.1.4 → dbt_copilot_python-0.1.5.dev2}/pyproject.toml

@@ -3,7 +3,7 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-copilot-python"
-version = "0.1.4"
+version = "0.1.5-dev2"
 description = "Helper functions to run Django and Flask applications in AWS Copilot/ECS."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 readme = "README.md"
@@ -11,17 +11,21 @@ packages = [{ include = "dbt_copilot_python" }]
 
 [tool.poetry.dependencies]
 python = "^3.9"
-requests = "^2.31.0"
+celery = "^5.3.6"
 opentelemetry-distro = "0.43b0"
 opentelemetry-exporter-otlp = "1.22.0"
 opentelemetry-instrumentation-wsgi = "0.43b0"
 opentelemetry-propagator-aws-xray = "^1.0.1"
 opentelemetry-sdk-extension-aws = "^2.0.1"
+requests = "^2.31.0"
+pytest-celery = "^0.0.0"
+redis = "^5.0.1"
+pytest-mock = "^3.12.0"
 
 [tool.poetry.group.dev.dependencies]
+pre-commit = "^3.3.3"
 pytest = "^7.3.2"
 requests-mock = "^1.11.0"
-pre-commit = "^3.3.3"
 tox = "^4.6.3"
 
 [build-system]