ct-mlflow-lib 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ from ct_mlflow_lib.utils import (
2
+ end_mlflow,
3
+ get_mlflow_run_id,
4
+ is_mlflow_active,
5
+ log_artifact,
6
+ log_metric,
7
+ log_metrics,
8
+ log_param,
9
+ log_params,
10
+ set_tag,
11
+ )
12
+ from ct_mlflow_lib.keras_callback import MLflowKerasCallback
13
+ from ct_mlflow_lib.product import try_init_product
14
+
15
+ __all__ = [
16
+ "try_init_product",
17
+ "is_mlflow_active",
18
+ "get_mlflow_run_id",
19
+ "end_mlflow",
20
+ "log_metric",
21
+ "log_metrics",
22
+ "log_param",
23
+ "log_params",
24
+ "set_tag",
25
+ "log_artifact",
26
+ "MLflowKerasCallback",
27
+ ]
@@ -0,0 +1,23 @@
1
+ """Cloudflare Access service-token headers for MLflow HTTP requests."""
2
+
3
+ import os
4
+
5
+ from mlflow.tracking.request_header.abstract_request_header_provider import (
6
+ RequestHeaderProvider,
7
+ )
8
+
9
+
10
+ class CloudflareAccessServiceTokenProvider(RequestHeaderProvider):
11
+ """Adds Cloudflare Access service-token headers when env vars are set."""
12
+
13
+ def in_context(self) -> bool:
14
+ return bool(
15
+ os.environ.get("CF_ACCESS_CLIENT_ID")
16
+ and os.environ.get("CF_ACCESS_CLIENT_SECRET")
17
+ )
18
+
19
+ def request_headers(self) -> dict[str, str]:
20
+ return {
21
+ "CF-Access-Client-Id": os.environ["CF_ACCESS_CLIENT_ID"],
22
+ "CF-Access-Client-Secret": os.environ["CF_ACCESS_CLIENT_SECRET"],
23
+ }
@@ -0,0 +1,52 @@
1
+ """Keras callback for logging epoch-level metrics to MLflow."""
2
+
3
+ try:
4
+ import tensorflow as tf
5
+
6
+ class MLflowKerasCallback(tf.keras.callbacks.Callback):
7
+ """Log Keras training metrics to MLflow at each epoch.
8
+
9
+ Use this callback in model.fit() to automatically log epoch-level
10
+ metrics (loss, accuracy, etc.) with an optional prefix.
11
+
12
+ Args:
13
+ prefix: Optional prefix for metric names, e.g. "autoencoder", "custom_model".
14
+ """
15
+
16
+ def __init__(self, prefix: str = ""):
17
+ super().__init__()
18
+ self.prefix = prefix
19
+
20
+ def on_epoch_end(self, epoch, logs=None):
21
+ """Called at the end of each epoch."""
22
+ if not logs:
23
+ return
24
+
25
+ try:
26
+ import mlflow
27
+
28
+ if not mlflow.active_run():
29
+ return
30
+
31
+ metrics = {}
32
+ for key, value in logs.items():
33
+ if self.prefix:
34
+ metric_name = f"{self.prefix}_{key}"
35
+ else:
36
+ metric_name = key
37
+ metrics[metric_name] = float(value)
38
+
39
+ mlflow.log_metrics(metrics, step=epoch)
40
+ except Exception:
41
+ pass
42
+
43
+ except ImportError:
44
+
45
+ class MLflowKerasCallback:
46
+ """Stub when TensorFlow is not installed."""
47
+
48
+ def __init__(self, prefix: str = ""):
49
+ raise ImportError(
50
+ "TensorFlow is required to use MLflowKerasCallback. "
51
+ "Install it with: pip install tensorflow"
52
+ )
@@ -0,0 +1,64 @@
1
+ """Product-specific MLflow initialization helpers."""
2
+
3
+ import os
4
+ import mlflow
5
+
6
+
7
+ def try_init_product(
8
+ task: str,
9
+ product: str,
10
+ env: str,
11
+ tags: dict | None = None,
12
+ ) -> tuple[bool, str | None]:
13
+ """Attempt to initialize MLflow for a product with graceful error reporting.
14
+
15
+ This is the recommended API for products that need graceful degradation when MLflow
16
+ is unavailable. It returns a status tuple instead of raising exceptions.
17
+
18
+ Args:
19
+ task: Task name only (e.g. ``"train"``). The experiment name is built as
20
+ ``{product}.{env}.{task}``.
21
+ product: Product name (e.g. ``"recommendation"``).
22
+ env: Deployment environment (e.g. ``"prod"``).
23
+ tags: Optional product-specific tags to add to the run.
24
+
25
+ Returns:
26
+ Tuple of (success: bool, reason: str | None).
27
+ - If success=True: MLflow is initialized and a run is active. reason=None.
28
+ - If success=False: MLflow init failed. reason contains the error reason.
29
+
30
+ Example:
31
+ success, reason = try_init_product("train", product="recommendation", env="prod", tags={"account_id": "123"})
32
+ if not success:
33
+ logger.warning(f"MLflow initialization failed: {reason}")
34
+ else:
35
+ import ct_mlflow_lib
36
+
37
+ ct_mlflow_lib.log_metric("auc", 0.87)
38
+ """
39
+ tracking_uri = os.environ.get("MLFLOW_TRACKING_URI")
40
+ if not tracking_uri:
41
+ return False, "MLFLOW_TRACKING_URI is not set"
42
+
43
+ experiment = f"{product}.{env}.{task}"
44
+
45
+ try:
46
+ mlflow.set_tracking_uri(tracking_uri)
47
+
48
+ # Create or get experiment
49
+ exp = mlflow.get_experiment_by_name(experiment)
50
+ if exp is None:
51
+ mlflow.create_experiment(experiment)
52
+ mlflow.set_experiment(experiment)
53
+
54
+ # Default tags + product-specific tags (passed to start_run only)
55
+ merged_tags = {"product": product, "env": env}
56
+ if tags:
57
+ merged_tags.update(tags)
58
+
59
+ mlflow.start_run(tags=merged_tags)
60
+
61
+ return True, None
62
+
63
+ except Exception as e:
64
+ return False, str(e)
ct_mlflow_lib/utils.py ADDED
@@ -0,0 +1,87 @@
1
+ from collections.abc import Callable
2
+ from typing import Any
3
+
4
+ import mlflow
5
+
6
+
7
+ def _run_when_active(action: Callable[[], None]) -> None:
8
+ """Run ``action`` only if a run is active; swallow exceptions."""
9
+ if not is_mlflow_active():
10
+ return
11
+ try:
12
+ action()
13
+ except Exception:
14
+ pass
15
+
16
+
17
+ def is_mlflow_active() -> bool:
18
+ """Check if MLflow tracking is currently active.
19
+
20
+ Returns:
21
+ True if a run is active, False otherwise.
22
+ """
23
+ return mlflow.active_run() is not None
24
+
25
+
26
+ def get_mlflow_run_id() -> str | None:
27
+ """Get the current active run ID.
28
+
29
+ Returns:
30
+ The run ID string, or None if no run is active.
31
+ """
32
+ run = mlflow.active_run()
33
+ return run.info.run_id if run else None
34
+
35
+
36
+ def log_metric(key: str, value: float, step: int | None = None) -> None:
37
+ """Log a single metric when a run is active; no-op otherwise."""
38
+ _run_when_active(lambda: mlflow.log_metric(key, value, step=step))
39
+
40
+
41
+ def log_metrics(metrics: dict[str, float], step: int | None = None) -> None:
42
+ """Log multiple metrics when a run is active; no-op otherwise."""
43
+ _run_when_active(lambda: mlflow.log_metrics(metrics, step=step))
44
+
45
+
46
+ def log_param(key: str, value: Any) -> None:
47
+ """Log a single parameter when a run is active; no-op otherwise."""
48
+ _run_when_active(lambda: mlflow.log_param(key, value))
49
+
50
+
51
+ def log_params(params: dict[str, Any]) -> None:
52
+ """Log multiple parameters when a run is active; no-op otherwise."""
53
+ _run_when_active(lambda: mlflow.log_params(params))
54
+
55
+
56
+ def set_tag(key: str, value: str) -> None:
57
+ """Set a tag when a run is active; no-op otherwise."""
58
+ _run_when_active(lambda: mlflow.set_tag(key, value))
59
+
60
+
61
+ def log_artifact(local_path: str) -> None:
62
+ """Log a local file as an artifact when a run is active; no-op otherwise."""
63
+ _run_when_active(lambda: mlflow.log_artifact(local_path))
64
+
65
+
66
+ def end_mlflow(exit_code: int | None = None, error_message: str | None = None) -> None:
67
+ """End the current MLflow run and log exit status.
68
+
69
+ Call this at the end of a job (success or failure) to log final status
70
+ and cleanly close the run.
71
+
72
+ Args:
73
+ exit_code: Optional exit code (0 = success, non-zero = failure).
74
+ error_message: Optional error message (truncated to 250 chars).
75
+ """
76
+
77
+ def _end_run() -> None:
78
+ if exit_code is not None:
79
+ mlflow.log_metric("exit_code", exit_code)
80
+ mlflow.set_tag("exit_status", "failed" if exit_code != 0 else "success")
81
+
82
+ if error_message:
83
+ mlflow.set_tag("error_message", str(error_message)[:250])
84
+
85
+ mlflow.end_run()
86
+
87
+ _run_when_active(_end_run)
@@ -0,0 +1,241 @@
1
+ Metadata-Version: 2.4
2
+ Name: ct-mlflow-lib
3
+ Version: 0.3.0
4
+ Summary: CleverTap internal MLflow client library
5
+ Project-URL: Repository, https://github.com/CleverTap-DS/MLflow
6
+ Requires-Python: >=3.11
7
+ Requires-Dist: databricks-sdk<1,>=0.38.0
8
+ Requires-Dist: gitpython<4,>=3.1.40
9
+ Requires-Dist: graphene<4,>=3.2.0
10
+ Requires-Dist: mlflow==3.11.1
11
+ Requires-Dist: opentelemetry-api<3,>=1.25.0
12
+ Requires-Dist: opentelemetry-sdk<3,>=1.25.0
13
+ Provides-Extra: dev
14
+ Requires-Dist: pytest-cov>=4.0; extra == 'dev'
15
+ Requires-Dist: pytest-xdist>=3.0; extra == 'dev'
16
+ Requires-Dist: pytest>=8.0; extra == 'dev'
17
+ Requires-Dist: tensorflow>=2.10; extra == 'dev'
18
+ Provides-Extra: keras
19
+ Requires-Dist: tensorflow>=2.10; extra == 'keras'
20
+ Description-Content-Type: text/markdown
21
+
22
+ # ct-mlflow-lib
23
+
24
+ Thin MLflow client library for CleverTap datascience products.
25
+
26
+ Application code should use **only** this package for tracking (metrics, params, artifacts).
27
+ Do not `import mlflow` in product code; use `ct_mlflow_lib` helpers so logging is a no-op when
28
+ no run is active.
29
+
30
+ ## Install
31
+
32
+ Pin a git tag in your `requirements.txt` or `pyproject.toml`:
33
+
34
+ ```bash
35
+ pip install "git+https://github.com/CleverTap-DS/MLflow@v0.1.0#subdirectory=ct_mlflow_lib"
36
+ ```
37
+
38
+ Or with `uv`:
39
+
40
+ ```bash
41
+ uv add "ct-mlflow-lib @ git+https://github.com/CleverTap-DS/MLflow@v0.1.0#subdirectory=ct_mlflow_lib"
42
+ ```
43
+
44
+ ### Optional: Keras support
45
+
46
+ To use `MLflowKerasCallback`, install with the `keras` extra:
47
+
48
+ ```bash
49
+ pip install "git+https://github.com/CleverTap-DS/MLflow@v0.1.0#subdirectory=ct_mlflow_lib[keras]"
50
+ ```
51
+
52
+ The core library works without TensorFlow. `MLflowKerasCallback` will raise an `ImportError` if TensorFlow is not installed.
53
+
54
+ ## Quick Start
55
+
56
+ The primary API is `try_init_product()`. Pass the **task**, **product**, and **env** names;
57
+ the experiment name is built as `{product}.{env}.{task}`.
58
+
59
+ ```python
60
+ import ct_mlflow_lib
61
+
62
+ success, reason = ct_mlflow_lib.try_init_product(
63
+ "train",
64
+ product="recommendation",
65
+ env="prod",
66
+ tags={
67
+ "account_id": "123",
68
+ "catalog_id": "456",
69
+ },
70
+ )
71
+
72
+ if not success:
73
+ logger.warning(f"MLflow disabled: {reason}")
74
+ else:
75
+ ct_mlflow_lib.log_metric("auc", 0.87)
76
+ ct_mlflow_lib.log_param("lr", 0.01)
77
+ ct_mlflow_lib.end_mlflow(exit_code=0)
78
+ ```
79
+
80
+ Logging helpers (`log_metric`, `log_metrics`, `log_param`, `log_params`, `set_tag`, `log_artifact`)
81
+ are safe to call even when init failed or no run is active: they no-op.
82
+
83
+ ## API Reference
84
+
85
+ ### `try_init_product(task: str, product: str, env: str, tags: dict | None = None) -> tuple[bool, str | None]`
86
+
87
+ Initialize MLflow for a product run. Returns immediately with status instead of raising exceptions.
88
+
89
+ **Args:**
90
+
91
+ - `task`: Task segment only (e.g. `"train"`, `"eval"`). Full experiment name is
92
+ `{product}.{env}.{task}` (e.g. `recommendation.prod.train`).
93
+ - `product`: Product name (e.g. `"recommendation"`, `"prediction"`).
94
+ - `env`: Deployment environment (e.g. `"prod"`, `"staging"`, `"dev"`).
95
+ - `tags`: Optional dict of product-specific tags (merged with defaults: `product`, `env`).
96
+
97
+ **Returns:** Tuple of `(success: bool, reason: str | None)`
98
+
99
+ - If `success=True`: MLflow is initialized and a run is active. `reason=None`.
100
+ - If `success=False`: MLflow init failed. `reason` explains why (e.g. `MLFLOW_TRACKING_URI is not set`,
101
+ or a connection error).
102
+
103
+ **Never raises.** Gracefully handles missing config, network errors, etc.
104
+
105
+ ### `log_metric`, `log_metrics`, `log_param`, `log_params`, `set_tag`, `log_artifact`
106
+
107
+ Delegate to MLflow when a run is active; otherwise no-op. Never raise.
108
+
109
+ ### `end_mlflow(exit_code: int | None = None, error_message: str | None = None) -> None`
110
+
111
+ End the current MLflow run and log exit status. Call this at the end of a job.
112
+
113
+ **Never raises.** Safe to call even if no run is active.
114
+
115
+ ```python
116
+ try:
117
+ ct_mlflow_lib.log_metric("auc", 0.87)
118
+ ct_mlflow_lib.end_mlflow(exit_code=0)
119
+ except Exception as e:
120
+ ct_mlflow_lib.end_mlflow(exit_code=1, error_message=str(e))
121
+ ```
122
+
123
+ ### `is_mlflow_active() -> bool`
124
+
125
+ Check if a run is currently active.
126
+
127
+ ### `get_mlflow_run_id() -> str | None`
128
+
129
+ Get the current run ID, or `None` if no run is active.
130
+
131
+ ### `MLflowKerasCallback(prefix: str = "")`
132
+
133
+ Keras callback for logging epoch-level metrics to MLflow. Use with `model.fit()`.
134
+
135
+ ```python
136
+ import tensorflow as tf
137
+ from ct_mlflow_lib import MLflowKerasCallback
138
+
139
+ model = tf.keras.Sequential([...])
140
+ model.fit(
141
+ x_train, y_train,
142
+ epochs=10,
143
+ callbacks=[MLflowKerasCallback(prefix="model")],
144
+ )
145
+ ```
146
+
147
+ ## Experiment naming convention
148
+
149
+ The experiment name is always `{product}.{env}.{task}`.
150
+
151
+ | product | env | task | Experiment name |
152
+ |---------|-----|------|-----------------|
153
+ | recommendation | prod | train | `recommendation.prod.train` |
154
+ | prediction | staging | train | `prediction.staging.train` |
155
+
156
+ ## Required environment variables
157
+
158
+ Set these in ECS task definitions, Batch job definitions, EC2 launch templates, or local `.env`.
159
+
160
+ | Variable | Required for init | Description | Example |
161
+ |----------|-------------------|-------------|---------|
162
+ | `MLFLOW_TRACKING_URI` | Yes | MLflow server URL (no `#` fragment) | `https://mlflow.example.com` |
163
+
164
+ If `MLFLOW_TRACKING_URI` is missing, `try_init_product()` returns `(False, reason)` instead of raising.
165
+
166
+ ## Cloudflare Access (production)
167
+
168
+ When the tracking server is behind Cloudflare Access, batch jobs must send a **service token** on
169
+ every HTTP request. This library registers an MLflow `RequestHeaderProvider` entry point that
170
+ adds the headers when credentials are present.
171
+
172
+ | Variable | When | Description |
173
+ |----------|------|-------------|
174
+ | `CF_ACCESS_CLIENT_ID` | Production behind Access | Service token client ID |
175
+ | `CF_ACCESS_CLIENT_SECRET` | Production behind Access | Service token client secret |
176
+
177
+ If either variable is unset, the provider does not inject headers (safe for local dev or servers not behind Access).
178
+
179
+ ## Error handling examples
180
+
181
+ ### Example 1: MLflow is optional (typical case)
182
+
183
+ ```python
184
+ import ct_mlflow_lib
185
+
186
+ success, reason = ct_mlflow_lib.try_init_product(
187
+ "train",
188
+ product="recommendation",
189
+ env="prod",
190
+ tags={"account_id": account_id},
191
+ )
192
+
193
+ if not success:
194
+ logger.warning(f"MLflow not available ({reason}), training without tracking")
195
+ else:
196
+ logger.info(f"MLflow tracking enabled (run: {ct_mlflow_lib.get_mlflow_run_id()})")
197
+
198
+ # Safe without guards: helpers no-op when inactive
199
+ ct_mlflow_lib.log_metric("auc", 0.87)
200
+ ct_mlflow_lib.end_mlflow(exit_code=0)
201
+ ```
202
+
203
+ ### Example 2: Wrap in product-specific helper
204
+
205
+ ```python
206
+ import ct_mlflow_lib
207
+
208
+ def init_mlflow_for_recommendation(account_id, catalog_id, recommendation_id, env):
209
+ success, reason = ct_mlflow_lib.try_init_product(
210
+ "train",
211
+ product="recommendation",
212
+ env=env,
213
+ tags={
214
+ "account_id": str(account_id),
215
+ "catalog_id": str(catalog_id),
216
+ "recommendation_id": str(recommendation_id),
217
+ },
218
+ )
219
+ if not success:
220
+ logger.warning(f"MLflow disabled: {reason}")
221
+ return success
222
+ ```
223
+
224
+ ## Development
225
+
226
+ Run `pre-commit install` before coding.
227
+
228
+ Install dev dependencies, then run tests:
229
+
230
+ ```bash
231
+ cd ct_mlflow_lib
232
+ uv sync --group dev
233
+ pytest tests/
234
+ ```
235
+
236
+ With pip only (no uv), use the optional `dev` extra:
237
+
238
+ ```bash
239
+ pip install -e ".[dev]"
240
+ pytest tests/
241
+ ```
@@ -0,0 +1,9 @@
1
+ ct_mlflow_lib/__init__.py,sha256=pDXtlvXER-sRDuhIRVipZwDYLnF2gkO_htgIgH888Go,544
2
+ ct_mlflow_lib/cloudflare_auth.py,sha256=6DLKPSAIoHN8ag1RYZN2M4LvfZ7rYcjgInQsQShCFNE,737
3
+ ct_mlflow_lib/keras_callback.py,sha256=lm5wf7ZGLCRxJUCuEDWu51dHBpRPT5nO0HS7SVkRdno,1592
4
+ ct_mlflow_lib/product.py,sha256=vgJVDTrpxQVZFBC-Ig3OVw4u1NvOgjVsG62BUUycrhs,2088
5
+ ct_mlflow_lib/utils.py,sha256=BtuJnhMz8G4F62Mkctk2KUHo2OiHNCew_xraCKTsRS8,2657
6
+ ct_mlflow_lib-0.3.0.dist-info/METADATA,sha256=-I-4agXq5BEK4M-NjrvTXqQEO178BY3VZEW_IzjuWZg,7285
7
+ ct_mlflow_lib-0.3.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
8
+ ct_mlflow_lib-0.3.0.dist-info/entry_points.txt,sha256=y5QNC_f1BCHMZ2X2IvqH47LAXD8eLy4m2KHmue1KoaM,113
9
+ ct_mlflow_lib-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.29.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [mlflow.request_header_provider]
2
+ cloudflare = ct_mlflow_lib.cloudflare_auth:CloudflareAccessServiceTokenProvider