chartreuse 6.3.0.dev0__tar.gz → 7.0.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chartreuse-6.3.0.dev0/src/chartreuse.egg-info → chartreuse-7.0.0.dev1}/PKG-INFO +8 -5
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/README.md +1 -1
- chartreuse-7.0.0.dev1/VERSION +1 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/pyproject.toml +10 -5
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse/chartreuse.py +3 -6
- chartreuse-7.0.0.dev1/src/chartreuse/chartreuse_restore.py +34 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse/chartreuse_upgrade.py +3 -4
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse/config_loader.py +3 -3
- chartreuse-7.0.0.dev1/src/chartreuse/kubernetes_helper.py +375 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse/utils/alembic_migration_helper.py +2 -1
- chartreuse-7.0.0.dev1/src/chartreuse/utils/command.py +40 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1/src/chartreuse.egg-info}/PKG-INFO +8 -5
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/SOURCES.txt +4 -1
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/entry_points.txt +1 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/requires.txt +2 -2
- chartreuse-6.3.0.dev0/VERSION +0 -1
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/LICENCE.md +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/MANIFEST.in +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/setup.cfg +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse/__init__.py +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse/utils/__init__.py +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/dependency_links.txt +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/top_level.txt +0 -0
- {chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/zip-safe +0 -0
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chartreuse
|
|
3
|
-
Version:
|
|
3
|
+
Version: 7.0.0.dev1
|
|
4
4
|
Summary: Helper for Alembic migrations within Kubernetes.
|
|
5
5
|
Author-email: wiremind <dev@wiremind.io>
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
7
7
|
Project-URL: github, https://github.com/wiremind/chartreuse
|
|
8
|
-
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Requires-Python: >=3.13.0
|
|
9
12
|
Description-Content-Type: text/markdown
|
|
10
13
|
License-File: LICENCE.md
|
|
11
14
|
Requires-Dist: alembic
|
|
12
|
-
Requires-Dist:
|
|
15
|
+
Requires-Dist: kubernetes
|
|
16
|
+
Requires-Dist: psycopg2
|
|
13
17
|
Requires-Dist: pydantic>=2.0.0
|
|
14
18
|
Requires-Dist: PyYAML
|
|
15
|
-
Requires-Dist: wiremind-kubernetes~=7.0
|
|
16
19
|
Provides-Extra: test
|
|
17
20
|
Requires-Dist: mock; extra == "test"
|
|
18
21
|
Requires-Dist: pytest; extra == "test"
|
|
@@ -40,7 +43,7 @@ Chartreuse leverages [Helm Hooks](https://helm.sh/docs/topics/charts_hooks/), th
|
|
|
40
43
|
|
|
41
44
|
### Requirements
|
|
42
45
|
|
|
43
|
-
- Python >= 3.
|
|
46
|
+
- Python >= 3.13
|
|
44
47
|
- Using Helm to deploy you application
|
|
45
48
|
- This Python package requires the `expecteddeploymentscales.wiremind.io` Kubernetes `Custom Resource Definition`:
|
|
46
49
|
|
|
@@ -11,7 +11,7 @@ Chartreuse leverages [Helm Hooks](https://helm.sh/docs/topics/charts_hooks/), th
|
|
|
11
11
|
|
|
12
12
|
### Requirements
|
|
13
13
|
|
|
14
|
-
- Python >= 3.
|
|
14
|
+
- Python >= 3.13
|
|
15
15
|
- Using Helm to deploy you application
|
|
16
16
|
- This Python package requires the `expecteddeploymentscales.wiremind.io` Kubernetes `Custom Resource Definition`:
|
|
17
17
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
7.0.0.dev1
|
|
@@ -6,15 +6,20 @@ readme = "README.md"
|
|
|
6
6
|
authors = [{ name = "wiremind", email = "dev@wiremind.io" }]
|
|
7
7
|
license="LGPL-3.0-or-later"
|
|
8
8
|
urls = { github = "https://github.com/wiremind/chartreuse"}
|
|
9
|
-
scripts = {chartreuse-upgrade = "chartreuse.chartreuse_upgrade:main"}
|
|
10
|
-
requires-python = ">=3.
|
|
9
|
+
scripts = {chartreuse-upgrade = "chartreuse.chartreuse_upgrade:main", chartreuse-restore = "chartreuse.chartreuse_restore:main"}
|
|
10
|
+
requires-python = ">=3.13.0"
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
"Programming Language :: Python :: 3.13",
|
|
14
|
+
"Programming Language :: Python :: 3.14",
|
|
15
|
+
]
|
|
11
16
|
|
|
12
17
|
dependencies = [
|
|
13
18
|
"alembic",
|
|
14
|
-
"
|
|
19
|
+
"kubernetes",
|
|
20
|
+
"psycopg2",
|
|
15
21
|
"pydantic>=2.0.0",
|
|
16
22
|
"PyYAML",
|
|
17
|
-
"wiremind-kubernetes~=7.0",
|
|
18
23
|
]
|
|
19
24
|
|
|
20
25
|
[build-system]
|
|
@@ -50,7 +55,7 @@ dev = [
|
|
|
50
55
|
|
|
51
56
|
[tool.ruff]
|
|
52
57
|
line-length = 120
|
|
53
|
-
target-version = "
|
|
58
|
+
target-version = "py313"
|
|
54
59
|
|
|
55
60
|
[tool.ruff.lint]
|
|
56
61
|
select = [
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
|
-
import wiremind_kubernetes.kubernetes_helper
|
|
4
|
-
|
|
5
3
|
from .config_loader import DatabaseConfig
|
|
4
|
+
from .kubernetes_helper import KubernetesDeploymentManager
|
|
6
5
|
from .utils import AlembicMigrationHelper
|
|
7
6
|
|
|
8
7
|
logger = logging.getLogger(__name__)
|
|
@@ -23,7 +22,7 @@ class Chartreuse:
|
|
|
23
22
|
self,
|
|
24
23
|
databases_config: dict[str, DatabaseConfig],
|
|
25
24
|
release_name: str,
|
|
26
|
-
kubernetes_helper:
|
|
25
|
+
kubernetes_helper: KubernetesDeploymentManager | None = None,
|
|
27
26
|
):
|
|
28
27
|
configure_logging()
|
|
29
28
|
|
|
@@ -56,9 +55,7 @@ class Chartreuse:
|
|
|
56
55
|
if kubernetes_helper:
|
|
57
56
|
self.kubernetes_helper = kubernetes_helper
|
|
58
57
|
else:
|
|
59
|
-
self.kubernetes_helper =
|
|
60
|
-
use_kubeconfig=None, release_name=release_name
|
|
61
|
-
)
|
|
58
|
+
self.kubernetes_helper = KubernetesDeploymentManager(use_kubeconfig=None, release_name=release_name)
|
|
62
59
|
|
|
63
60
|
@property
|
|
64
61
|
def is_migration_needed(self) -> bool:
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from .chartreuse_upgrade import ensure_safe_run
|
|
5
|
+
from .kubernetes_helper import KubernetesDeploymentManager
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main() -> None:
|
|
11
|
+
"""
|
|
12
|
+
Restore Deployments that a pre-deployment migration (CHARTREUSE_UPGRADE_BEFORE_DEPLOYMENT)
|
|
13
|
+
stopped and that the deployment itself could not scale back up.
|
|
14
|
+
|
|
15
|
+
On that path chartreuse-upgrade intentionally skips start_pods() and relies on the
|
|
16
|
+
deployment that follows to restore replicas. That works for Deployments whose chart sets
|
|
17
|
+
spec.replicas, but HPA-managed ones omit it: they stay at 0 replicas and an HPA whose
|
|
18
|
+
target is at 0 with minReplicas >= 1 is ScalingDisabled and never scales them back up.
|
|
19
|
+
|
|
20
|
+
Runs as a post-deployment (e.g. ArgoCD PostSync) hook: the new pod template is already
|
|
21
|
+
applied, so scaling up only ever starts new-version pods. Only Deployments still carrying
|
|
22
|
+
the annotation set by stop_pods() are touched. If this program fails, the deployment is
|
|
23
|
+
considered as failed: stranded workers must be visible, not a log line.
|
|
24
|
+
"""
|
|
25
|
+
ensure_safe_run()
|
|
26
|
+
|
|
27
|
+
release_name: str = os.environ["CHARTREUSE_RELEASE_NAME"]
|
|
28
|
+
|
|
29
|
+
deployment_manager = KubernetesDeploymentManager(release_name=release_name, use_kubeconfig=None)
|
|
30
|
+
deployment_manager.restore_stopped_pods()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
if __name__ == "__main__":
|
|
34
|
+
main()
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
|
-
from wiremind_kubernetes import KubernetesDeploymentManager
|
|
5
|
-
|
|
6
4
|
from chartreuse import get_version
|
|
7
5
|
|
|
8
6
|
from .chartreuse import Chartreuse
|
|
9
7
|
from .config_loader import load_multi_database_config
|
|
8
|
+
from .kubernetes_helper import KubernetesDeploymentManager
|
|
10
9
|
|
|
11
10
|
logger = logging.getLogger(__name__)
|
|
12
11
|
|
|
@@ -20,10 +19,10 @@ def ensure_safe_run() -> None:
|
|
|
20
19
|
# Get "1.2" from "1.2.3"
|
|
21
20
|
package_v_major_minor: list[str] = package_v.split(".", 2)[:2]
|
|
22
21
|
|
|
23
|
-
helm_chart_v: str = os.getenv("
|
|
22
|
+
helm_chart_v: str = os.getenv("HELM_CHART_APP_VERSION", "")
|
|
24
23
|
if not helm_chart_v:
|
|
25
24
|
raise ValueError(
|
|
26
|
-
"Couldn't get the Chartreuse's Helm Chart
|
|
25
|
+
"Couldn't get the Chartreuse's Helm Chart appVersion from the env var HELM_CHART_APP_VERSION,"
|
|
27
26
|
" couldn't make sure that the package is of a compatible version, ABORTING!"
|
|
28
27
|
)
|
|
29
28
|
helm_chart_v_major_minor: list[str] = helm_chart_v.split(".", 2)[:2]
|
|
@@ -39,7 +39,7 @@ class DatabaseConfig(BaseModel):
|
|
|
39
39
|
@classmethod
|
|
40
40
|
def validate_dialect(cls, v: str) -> str:
|
|
41
41
|
"""Validate database dialect."""
|
|
42
|
-
supported_dialects = ["postgresql
|
|
42
|
+
supported_dialects = ["postgresql", "mysql", "sqlite", "oracle", "mssql", "clickhouse"]
|
|
43
43
|
if v.lower() not in supported_dialects:
|
|
44
44
|
logger.warning("Dialect '%s' might not be supported. Supported dialects: %s", v, supported_dialects)
|
|
45
45
|
return v
|
|
@@ -86,7 +86,7 @@ def load_multi_database_config(config_path: str) -> dict[str, DatabaseConfig]:
|
|
|
86
86
|
```yaml
|
|
87
87
|
databases:
|
|
88
88
|
main: # Database name/identifier
|
|
89
|
-
dialect: postgresql
|
|
89
|
+
dialect: postgresql
|
|
90
90
|
user: app_user
|
|
91
91
|
password: app_password
|
|
92
92
|
host: postgres-main
|
|
@@ -99,7 +99,7 @@ def load_multi_database_config(config_path: str) -> dict[str, DatabaseConfig]:
|
|
|
99
99
|
|
|
100
100
|
# For single database setups, just include one database:
|
|
101
101
|
# analytics:
|
|
102
|
-
# dialect: postgresql
|
|
102
|
+
# dialect: postgresql
|
|
103
103
|
# user: analytics_user
|
|
104
104
|
# password: analytics_password
|
|
105
105
|
# host: postgres-analytics
|
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Kubernetes helpers to stop, start and restore the application's Deployments around a database
|
|
3
|
+
migration.
|
|
4
|
+
|
|
5
|
+
Trimmed port of wiremind_kubernetes.KubernetesDeploymentManager (a library being
|
|
6
|
+
decommissioned): only what chartreuse uses survived — ExpectedDeploymentScale-driven
|
|
7
|
+
scale down/up with HPA disabling, plus the stopped-by annotation and restore logic.
|
|
8
|
+
Depends only on the official kubernetes client.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import functools
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import pprint
|
|
15
|
+
import time
|
|
16
|
+
from collections.abc import Callable, Generator
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import kubernetes
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
HPA_ID_PREFIX = "wm--disabled--kube"
|
|
24
|
+
# Set by stop_pods() on every Deployment it scales down, cleared by start_pods() and
|
|
25
|
+
# restore_stopped_pods(). Allows telling "stopped by us" apart from "deliberately scaled to 0".
|
|
26
|
+
STOPPED_ANNOTATION = "wiremind.io/stopped-by"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_kubernetes_config(use_kubeconfig: bool | None = None) -> None:
|
|
30
|
+
"""
|
|
31
|
+
Load kubernetes configuration in memory, either from incluster method or from kubeconfig.
|
|
32
|
+
:param use_kubeconfig:
|
|
33
|
+
If True: Use ~/.kube/config file to authenticate.
|
|
34
|
+
If False: use kubernetes built-in incluster mechanism.
|
|
35
|
+
If None, will try to load built-in incluster mechanism, then try config file.
|
|
36
|
+
Defaults to None.
|
|
37
|
+
"""
|
|
38
|
+
if os.environ.get("CLASSIC_K8S_CONFIG"):
|
|
39
|
+
# We are in a Kind cluster for E2E test! Never use in cluster config.
|
|
40
|
+
kubernetes.config.load_kube_config()
|
|
41
|
+
return
|
|
42
|
+
|
|
43
|
+
if use_kubeconfig is True:
|
|
44
|
+
kubernetes.config.load_kube_config()
|
|
45
|
+
elif use_kubeconfig is False:
|
|
46
|
+
kubernetes.config.load_incluster_config()
|
|
47
|
+
elif use_kubeconfig is None:
|
|
48
|
+
if os.path.exists(kubernetes.config.incluster_config.SERVICE_TOKEN_FILENAME):
|
|
49
|
+
kubernetes.config.load_incluster_config()
|
|
50
|
+
else:
|
|
51
|
+
kubernetes.config.load_kube_config()
|
|
52
|
+
logger.debug("Kubernetes configuration successfully set.")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def retry_kubernetes_request(function: Callable) -> Callable:
|
|
56
|
+
"""
|
|
57
|
+
Decorator that retries a failed Kubernetes API request if needed and ignores 404
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
@functools.wraps(function)
|
|
61
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
62
|
+
try:
|
|
63
|
+
return function(*args, **kwargs)
|
|
64
|
+
except kubernetes.client.rest.ApiException as e:
|
|
65
|
+
if e.status == 404:
|
|
66
|
+
logger.warning("Not found, ignoring.")
|
|
67
|
+
return
|
|
68
|
+
logger.error(e)
|
|
69
|
+
logger.info("Retrying in 5 seconds...")
|
|
70
|
+
time.sleep(5)
|
|
71
|
+
return function(*args, **kwargs)
|
|
72
|
+
|
|
73
|
+
return wrapper
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def retry_kubernetes_request_no_ignore(function: Callable) -> Callable:
|
|
77
|
+
"""
|
|
78
|
+
Decorator that retries a failed Kubernetes API request if needed and do NOT ignore 404 (raise if 404)
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
@functools.wraps(function)
|
|
82
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
83
|
+
try:
|
|
84
|
+
return function(*args, **kwargs)
|
|
85
|
+
except kubernetes.client.rest.ApiException as e:
|
|
86
|
+
if e.status == 404:
|
|
87
|
+
raise
|
|
88
|
+
logger.error(e)
|
|
89
|
+
logger.info("Retrying in 5 seconds...")
|
|
90
|
+
time.sleep(5)
|
|
91
|
+
return function(*args, **kwargs)
|
|
92
|
+
|
|
93
|
+
return wrapper
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _get_namespace_from_kube() -> str:
|
|
97
|
+
return open("/var/run/secrets/kubernetes.io/serviceaccount/namespace").read()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class KubernetesDeploymentManager:
|
|
101
|
+
"""
|
|
102
|
+
Scale down/up all Deployments that should be stopped/started when doing database
|
|
103
|
+
migration/maintenance (alembic, dump, etc). Each managed Deployment must define an
|
|
104
|
+
ExpectedDeploymentScale.
|
|
105
|
+
|
|
106
|
+
Usage:
|
|
107
|
+
manager = KubernetesDeploymentManager(release_name="my-release", use_kubeconfig=None)
|
|
108
|
+
manager.stop_pods()
|
|
109
|
+
do_something('wololo')
|
|
110
|
+
manager.start_pods()
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
SCALE_DOWN_MAX_WAIT_TIME: int = 3600
|
|
114
|
+
|
|
115
|
+
def __init__(
|
|
116
|
+
self,
|
|
117
|
+
release_name: str,
|
|
118
|
+
use_kubeconfig: bool | None = False,
|
|
119
|
+
namespace: str | None = None,
|
|
120
|
+
should_load_kubernetes_config: bool = True,
|
|
121
|
+
):
|
|
122
|
+
if should_load_kubernetes_config:
|
|
123
|
+
load_kubernetes_config(use_kubeconfig=use_kubeconfig)
|
|
124
|
+
self.release_name = release_name
|
|
125
|
+
self.namespace = namespace if namespace else _get_namespace_from_kube()
|
|
126
|
+
self.client_appsv1_api = kubernetes.client.AppsV1Api()
|
|
127
|
+
self.client_corev1_api = kubernetes.client.CoreV1Api()
|
|
128
|
+
self.client_autoscalingv2_api = kubernetes.client.AutoscalingV2Api()
|
|
129
|
+
self.client_custom_objects_api = kubernetes.client.CustomObjectsApi()
|
|
130
|
+
|
|
131
|
+
def get_deployment_scale(self, deployment_name: str) -> kubernetes.client.V1Scale:
|
|
132
|
+
logger.debug("Getting deployment scale for %s", deployment_name)
|
|
133
|
+
return self.client_appsv1_api.read_namespaced_deployment_scale(deployment_name, self.namespace)
|
|
134
|
+
|
|
135
|
+
@retry_kubernetes_request
|
|
136
|
+
def scale_down_deployment(self, deployment_name: str) -> None:
|
|
137
|
+
body = self.get_deployment_scale(deployment_name)
|
|
138
|
+
logger.debug("Deleting all Pods for %s", deployment_name)
|
|
139
|
+
body.spec.replicas = 0
|
|
140
|
+
self.client_appsv1_api.patch_namespaced_deployment_scale(deployment_name, self.namespace, body)
|
|
141
|
+
logger.debug("Done deleting.")
|
|
142
|
+
|
|
143
|
+
@retry_kubernetes_request
|
|
144
|
+
def scale_up_deployment(self, deployment_name: str, pod_amount: int) -> None:
|
|
145
|
+
body = self.get_deployment_scale(deployment_name)
|
|
146
|
+
logger.debug("Recreating backend Pods for %s", deployment_name)
|
|
147
|
+
body.spec.replicas = pod_amount
|
|
148
|
+
self.client_appsv1_api.patch_namespaced_deployment_scale(deployment_name, self.namespace, body)
|
|
149
|
+
logger.debug("Done recreating.")
|
|
150
|
+
|
|
151
|
+
@retry_kubernetes_request_no_ignore
|
|
152
|
+
def _get_pods_from_deployment(self, deployment_name: str) -> list:
|
|
153
|
+
logger.debug("Asking if Deployment %s is stopped", deployment_name)
|
|
154
|
+
labels = self.client_appsv1_api.read_namespaced_deployment(
|
|
155
|
+
deployment_name, self.namespace
|
|
156
|
+
).spec.selector.match_labels
|
|
157
|
+
|
|
158
|
+
try:
|
|
159
|
+
return self.client_corev1_api.list_namespaced_pod(
|
|
160
|
+
namespace=self.namespace,
|
|
161
|
+
label_selector=",".join(["{}={}".format(*kv) for kv in labels.items()]),
|
|
162
|
+
).items
|
|
163
|
+
except kubernetes.client.rest.ApiException as e:
|
|
164
|
+
if e.status == 404:
|
|
165
|
+
return []
|
|
166
|
+
else:
|
|
167
|
+
raise
|
|
168
|
+
|
|
169
|
+
def is_deployment_stopped(self, deployment_name: str) -> bool:
|
|
170
|
+
try:
|
|
171
|
+
pod_list: list = self._get_pods_from_deployment(deployment_name)
|
|
172
|
+
except kubernetes.client.rest.ApiException as e:
|
|
173
|
+
if e.status == 404:
|
|
174
|
+
logger.warning("Not found, ignoring.")
|
|
175
|
+
return True
|
|
176
|
+
raise
|
|
177
|
+
|
|
178
|
+
current_scale = 0
|
|
179
|
+
for pod in pod_list:
|
|
180
|
+
if pod.status.phase not in ("Failed"):
|
|
181
|
+
current_scale += 1
|
|
182
|
+
|
|
183
|
+
if current_scale > 0:
|
|
184
|
+
logger.info("%s Deployment has %s living replicas", deployment_name, current_scale)
|
|
185
|
+
return False
|
|
186
|
+
return True
|
|
187
|
+
|
|
188
|
+
def get_deployment_hpa(self, *, deployment_name: str) -> Generator:
|
|
189
|
+
for hpa in self.client_autoscalingv2_api.list_namespaced_horizontal_pod_autoscaler(self.namespace).items:
|
|
190
|
+
if hpa.spec.scale_target_ref.kind == "Deployment" and hpa.spec.scale_target_ref.name == deployment_name:
|
|
191
|
+
yield hpa
|
|
192
|
+
|
|
193
|
+
def patch_deployment_hpa(self, *, hpa_name: str, body: Any) -> None:
|
|
194
|
+
self.client_autoscalingv2_api.patch_namespaced_horizontal_pod_autoscaler(
|
|
195
|
+
name=hpa_name, namespace=self.namespace, body=body
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
@retry_kubernetes_request
|
|
199
|
+
def annotate_deployment(self, deployment_name: str, annotations: dict[str, str | None]) -> None:
|
|
200
|
+
"""
|
|
201
|
+
Add or update annotations on a Deployment. A None value removes the annotation.
|
|
202
|
+
"""
|
|
203
|
+
body = {"metadata": {"annotations": annotations}}
|
|
204
|
+
self.client_appsv1_api.patch_namespaced_deployment(deployment_name, self.namespace, body)
|
|
205
|
+
|
|
206
|
+
@retry_kubernetes_request_no_ignore
|
|
207
|
+
def _get_expected_deployment_scale_dict(self) -> dict[int, dict[str, int]]:
|
|
208
|
+
"""
|
|
209
|
+
Return a dict of expected deployment scale:
|
|
210
|
+
{
|
|
211
|
+
0: { # priority
|
|
212
|
+
# key: Deployment name, only if it has an associated eds
|
|
213
|
+
# value: expected Deployment Scale (replicas)
|
|
214
|
+
"my-deployment": 3,
|
|
215
|
+
"my-other-deployment": 42
|
|
216
|
+
},
|
|
217
|
+
1: {
|
|
218
|
+
"my-third-deployment": 17,
|
|
219
|
+
},
|
|
220
|
+
}
|
|
221
|
+
"""
|
|
222
|
+
logger.debug("Getting Expected Deployment Scale list")
|
|
223
|
+
eds_list: list[dict[str, Any]] = []
|
|
224
|
+
release_label_keys = ["app.kubernetes.io/instance", "release"]
|
|
225
|
+
|
|
226
|
+
for release_label_key in release_label_keys:
|
|
227
|
+
logger.debug(f"Getting Expected Deployment Scale list with the release label key {release_label_key}")
|
|
228
|
+
try:
|
|
229
|
+
eds_list.extend(
|
|
230
|
+
self.client_custom_objects_api.list_namespaced_custom_object(
|
|
231
|
+
namespace=self.namespace,
|
|
232
|
+
group="wiremind.io",
|
|
233
|
+
version="v1",
|
|
234
|
+
plural="expecteddeploymentscales",
|
|
235
|
+
label_selector=f"{release_label_key}={self.release_name}",
|
|
236
|
+
)["items"]
|
|
237
|
+
)
|
|
238
|
+
except kubernetes.client.rest.ApiException as e:
|
|
239
|
+
if e.status != 404:
|
|
240
|
+
raise
|
|
241
|
+
|
|
242
|
+
eds_dict: dict[int, dict[str, int]] = {}
|
|
243
|
+
for eds in eds_list:
|
|
244
|
+
deployment_name: str = eds["spec"]["deploymentName"]
|
|
245
|
+
expected_scale: int = eds["spec"]["expectedScale"]
|
|
246
|
+
priority: int = eds["spec"].get("priority", 0)
|
|
247
|
+
|
|
248
|
+
if priority not in eds_dict:
|
|
249
|
+
eds_dict[priority] = {}
|
|
250
|
+
|
|
251
|
+
eds_dict[priority][deployment_name] = expected_scale
|
|
252
|
+
|
|
253
|
+
logger.debug("Deployments are %s", pprint.pformat(eds_dict))
|
|
254
|
+
return eds_dict
|
|
255
|
+
|
|
256
|
+
@retry_kubernetes_request
|
|
257
|
+
def disable_hpa(self, *, deployment_name: str) -> None:
|
|
258
|
+
for hpa in self.get_deployment_hpa(deployment_name=deployment_name):
|
|
259
|
+
# Tell the hpa to manage a non-existing Deployment
|
|
260
|
+
hpa.spec.scale_target_ref.name = f"{HPA_ID_PREFIX}-{deployment_name}"
|
|
261
|
+
self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa)
|
|
262
|
+
|
|
263
|
+
@retry_kubernetes_request
|
|
264
|
+
def re_enable_hpa(self, *, deployment_name: str) -> None:
|
|
265
|
+
for hpa in self.get_deployment_hpa(deployment_name=f"{HPA_ID_PREFIX}-{deployment_name}"):
|
|
266
|
+
hpa.spec.scale_target_ref.name = deployment_name
|
|
267
|
+
self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa)
|
|
268
|
+
|
|
269
|
+
def _are_deployments_stopped(self, deployment_dict: dict[str, int]) -> bool:
|
|
270
|
+
for deployment_name in deployment_dict:
|
|
271
|
+
if not self.is_deployment_stopped(deployment_name):
|
|
272
|
+
return False
|
|
273
|
+
return True
|
|
274
|
+
|
|
275
|
+
def _stop_deployments(self, deployment_dict: dict[str, int]) -> None:
|
|
276
|
+
"""
|
|
277
|
+
Scale down a dict (deployment_name, expected_scale) of Deployments.
|
|
278
|
+
"""
|
|
279
|
+
for deployment_name in deployment_dict:
|
|
280
|
+
self.annotate_deployment(deployment_name, {STOPPED_ANNOTATION: self.release_name})
|
|
281
|
+
for _ in range(self.SCALE_DOWN_MAX_WAIT_TIME):
|
|
282
|
+
for deployment_name in deployment_dict:
|
|
283
|
+
self.disable_hpa(deployment_name=deployment_name)
|
|
284
|
+
self.scale_down_deployment(deployment_name)
|
|
285
|
+
if self._are_deployments_stopped(deployment_dict):
|
|
286
|
+
break
|
|
287
|
+
time.sleep(1)
|
|
288
|
+
else:
|
|
289
|
+
raise Exception("Timed out waiting for pods to be deleted: aborting.")
|
|
290
|
+
|
|
291
|
+
def stop_pods(self) -> None:
|
|
292
|
+
"""
|
|
293
|
+
Scale to 0 all deployments for which an ExpectedDeploymentScale links to.
|
|
294
|
+
stop all deployments, then wait for actual stop, by priority (descending order):
|
|
295
|
+
Example: stop all deployments with priority 1, then all deployments with priority 0
|
|
296
|
+
"""
|
|
297
|
+
expected_deployment_scale_dict: dict[int, dict[str, int]] = self._get_expected_deployment_scale_dict()
|
|
298
|
+
|
|
299
|
+
logger.info("Scaling down application Deployments...")
|
|
300
|
+
if not expected_deployment_scale_dict:
|
|
301
|
+
logger.info("No Deployments to scale down")
|
|
302
|
+
return
|
|
303
|
+
|
|
304
|
+
priorities: list[int] = sorted(expected_deployment_scale_dict, reverse=True)
|
|
305
|
+
for priority in priorities:
|
|
306
|
+
priority_dict: dict[str, int] = expected_deployment_scale_dict[priority]
|
|
307
|
+
if len(priority_dict):
|
|
308
|
+
self._stop_deployments(priority_dict)
|
|
309
|
+
logger.info("Done scaling down application Deployments.")
|
|
310
|
+
|
|
311
|
+
def start_pods(self) -> None:
|
|
312
|
+
"""
|
|
313
|
+
Start all Pods that should be started
|
|
314
|
+
"""
|
|
315
|
+
expected_deployment_scale_dict: dict[int, dict[str, int]] = self._get_expected_deployment_scale_dict()
|
|
316
|
+
|
|
317
|
+
logger.info("Scaling up application Deployments...")
|
|
318
|
+
if not expected_deployment_scale_dict:
|
|
319
|
+
logger.info("No Deployments to scale up")
|
|
320
|
+
return
|
|
321
|
+
|
|
322
|
+
scaled: bool = False
|
|
323
|
+
for priority_dict in expected_deployment_scale_dict.values():
|
|
324
|
+
if len(priority_dict):
|
|
325
|
+
scaled = True
|
|
326
|
+
for name, expected_scale in priority_dict.items():
|
|
327
|
+
self.re_enable_hpa(deployment_name=name)
|
|
328
|
+
self.scale_up_deployment(name, expected_scale)
|
|
329
|
+
self.annotate_deployment(name, {STOPPED_ANNOTATION: None})
|
|
330
|
+
if scaled:
|
|
331
|
+
logger.info("Done scaling up application Deployments")
|
|
332
|
+
else:
|
|
333
|
+
logger.info("No Deployments to scale up")
|
|
334
|
+
|
|
335
|
+
def restore_stopped_pods(self) -> None:
|
|
336
|
+
"""
|
|
337
|
+
Restore Deployments that stop_pods() scaled down and that nothing scaled back up since.
|
|
338
|
+
|
|
339
|
+
Only Deployments still carrying STOPPED_ANNOTATION are touched, so a Deployment
|
|
340
|
+
deliberately scaled to zero outside of stop_pods() is left alone.
|
|
341
|
+
|
|
342
|
+
Meant to run after the deployment that follows a stop_pods() without start_pods()
|
|
343
|
+
(CHARTREUSE_UPGRADE_BEFORE_DEPLOYMENT): that deployment restores spec.replicas of
|
|
344
|
+
Deployments that define it in their chart, but HPA-managed ones omit it, and an HPA
|
|
345
|
+
whose target is at 0 replicas with minReplicas >= 1 is ScalingDisabled and will never
|
|
346
|
+
scale it back up.
|
|
347
|
+
"""
|
|
348
|
+
expected_deployment_scale_dict: dict[int, dict[str, int]] = self._get_expected_deployment_scale_dict()
|
|
349
|
+
|
|
350
|
+
logger.info("Restoring Deployments left stopped...")
|
|
351
|
+
restored: list[str] = []
|
|
352
|
+
for priority in sorted(expected_deployment_scale_dict):
|
|
353
|
+
for name, expected_scale in expected_deployment_scale_dict[priority].items():
|
|
354
|
+
try:
|
|
355
|
+
deployment = self.client_appsv1_api.read_namespaced_deployment(name, self.namespace)
|
|
356
|
+
except kubernetes.client.rest.ApiException as e:
|
|
357
|
+
if e.status == 404:
|
|
358
|
+
continue
|
|
359
|
+
raise
|
|
360
|
+
if STOPPED_ANNOTATION not in (deployment.metadata.annotations or {}):
|
|
361
|
+
continue
|
|
362
|
+
self.re_enable_hpa(deployment_name=name)
|
|
363
|
+
if not deployment.spec.replicas:
|
|
364
|
+
target_scale = expected_scale
|
|
365
|
+
if target_scale < 1 and any(self.get_deployment_hpa(deployment_name=name)):
|
|
366
|
+
# Any scale >= 1 re-arms the HPA, which then enforces its own minReplicas
|
|
367
|
+
target_scale = 1
|
|
368
|
+
if target_scale > 0:
|
|
369
|
+
self.scale_up_deployment(name, target_scale)
|
|
370
|
+
self.annotate_deployment(name, {STOPPED_ANNOTATION: None})
|
|
371
|
+
restored.append(name)
|
|
372
|
+
if restored:
|
|
373
|
+
logger.info("Done restoring stopped Deployments: %s", ", ".join(restored))
|
|
374
|
+
else:
|
|
375
|
+
logger.info("No stopped Deployments to restore")
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import shlex
|
|
3
|
+
import subprocess
|
|
4
|
+
|
|
5
|
+
logger = logging.getLogger(__name__)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def run_command(command: list | str, return_result: bool = False, **kw_args) -> tuple[str, str, int]:
|
|
9
|
+
"""
|
|
10
|
+
Run command, print stdout/stderr, check that command exited correctly, return stdout/err
|
|
11
|
+
"""
|
|
12
|
+
logger.debug(f"Running {command}")
|
|
13
|
+
|
|
14
|
+
interpreted_command: list[str]
|
|
15
|
+
if isinstance(command, str):
|
|
16
|
+
interpreted_command = shlex.split(command)
|
|
17
|
+
else:
|
|
18
|
+
interpreted_command = command
|
|
19
|
+
|
|
20
|
+
process = subprocess.Popen(
|
|
21
|
+
interpreted_command,
|
|
22
|
+
stdout=subprocess.PIPE,
|
|
23
|
+
stderr=subprocess.STDOUT,
|
|
24
|
+
universal_newlines=True,
|
|
25
|
+
**kw_args,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
if return_result:
|
|
29
|
+
out, err = process.communicate()
|
|
30
|
+
return (out, err, process.returncode)
|
|
31
|
+
|
|
32
|
+
if process.stdout:
|
|
33
|
+
for line in iter(process.stdout.readline, ""):
|
|
34
|
+
logger.info(line.strip())
|
|
35
|
+
process.wait()
|
|
36
|
+
|
|
37
|
+
if process.returncode:
|
|
38
|
+
raise subprocess.CalledProcessError(process.returncode, command)
|
|
39
|
+
|
|
40
|
+
return "", "", 0
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: chartreuse
|
|
3
|
-
Version:
|
|
3
|
+
Version: 7.0.0.dev1
|
|
4
4
|
Summary: Helper for Alembic migrations within Kubernetes.
|
|
5
5
|
Author-email: wiremind <dev@wiremind.io>
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
7
7
|
Project-URL: github, https://github.com/wiremind/chartreuse
|
|
8
|
-
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Requires-Python: >=3.13.0
|
|
9
12
|
Description-Content-Type: text/markdown
|
|
10
13
|
License-File: LICENCE.md
|
|
11
14
|
Requires-Dist: alembic
|
|
12
|
-
Requires-Dist:
|
|
15
|
+
Requires-Dist: kubernetes
|
|
16
|
+
Requires-Dist: psycopg2
|
|
13
17
|
Requires-Dist: pydantic>=2.0.0
|
|
14
18
|
Requires-Dist: PyYAML
|
|
15
|
-
Requires-Dist: wiremind-kubernetes~=7.0
|
|
16
19
|
Provides-Extra: test
|
|
17
20
|
Requires-Dist: mock; extra == "test"
|
|
18
21
|
Requires-Dist: pytest; extra == "test"
|
|
@@ -40,7 +43,7 @@ Chartreuse leverages [Helm Hooks](https://helm.sh/docs/topics/charts_hooks/), th
|
|
|
40
43
|
|
|
41
44
|
### Requirements
|
|
42
45
|
|
|
43
|
-
- Python >= 3.
|
|
46
|
+
- Python >= 3.13
|
|
44
47
|
- Using Helm to deploy you application
|
|
45
48
|
- This Python package requires the `expecteddeploymentscales.wiremind.io` Kubernetes `Custom Resource Definition`:
|
|
46
49
|
|
|
@@ -5,8 +5,10 @@ VERSION
|
|
|
5
5
|
pyproject.toml
|
|
6
6
|
src/chartreuse/__init__.py
|
|
7
7
|
src/chartreuse/chartreuse.py
|
|
8
|
+
src/chartreuse/chartreuse_restore.py
|
|
8
9
|
src/chartreuse/chartreuse_upgrade.py
|
|
9
10
|
src/chartreuse/config_loader.py
|
|
11
|
+
src/chartreuse/kubernetes_helper.py
|
|
10
12
|
src/chartreuse.egg-info/PKG-INFO
|
|
11
13
|
src/chartreuse.egg-info/SOURCES.txt
|
|
12
14
|
src/chartreuse.egg-info/dependency_links.txt
|
|
@@ -15,4 +17,5 @@ src/chartreuse.egg-info/requires.txt
|
|
|
15
17
|
src/chartreuse.egg-info/top_level.txt
|
|
16
18
|
src/chartreuse.egg-info/zip-safe
|
|
17
19
|
src/chartreuse/utils/__init__.py
|
|
18
|
-
src/chartreuse/utils/alembic_migration_helper.py
|
|
20
|
+
src/chartreuse/utils/alembic_migration_helper.py
|
|
21
|
+
src/chartreuse/utils/command.py
|
chartreuse-6.3.0.dev0/VERSION
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
6.3.0-dev0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chartreuse-6.3.0.dev0 → chartreuse-7.0.0.dev1}/src/chartreuse.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|