controlvar 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {controlvar-0.2.0 → controlvar-0.3.0}/PKG-INFO +128 -25
- {controlvar-0.2.0 → controlvar-0.3.0}/README.md +127 -24
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/__init__.py +8 -1
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/attribution.py +7 -4
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/cache.py +14 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/client.py +253 -18
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/evaluator.py +97 -19
- controlvar-0.3.0/controlvar/exposure.py +52 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/local.py +8 -2
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/transport.py +24 -1
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/types.py +54 -3
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/version.py +1 -1
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar.egg-info/PKG-INFO +128 -25
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar.egg-info/SOURCES.txt +3 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/pyproject.toml +1 -1
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_client.py +545 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_contract_smoke.py +83 -1
- controlvar-0.3.0/tests/test_evaluator.py +382 -0
- controlvar-0.3.0/tests/test_event_id.py +131 -0
- controlvar-0.3.0/tests/test_exposure.py +617 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_local.py +340 -4
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_transport.py +26 -0
- controlvar-0.2.0/tests/test_evaluator.py +0 -214
- {controlvar-0.2.0 → controlvar-0.3.0}/LICENSE +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/hashing.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/py.typed +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar/testing.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar.egg-info/dependency_links.txt +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar.egg-info/requires.txt +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/controlvar.egg-info/top_level.txt +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/setup.cfg +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/setup.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_attribution.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_hashing.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_no_real_aws.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_package.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_readme.py +0 -0
- {controlvar-0.2.0 → controlvar-0.3.0}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: controlvar
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Python client for the Controlvar public API (experiments, feature flags, event tracking)
|
|
5
5
|
License: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/Controlvar/controlvar/tree/main/sdk/python#readme
|
|
@@ -42,11 +42,11 @@ else to install.
|
|
|
42
42
|
|
|
43
43
|
If you installed `experimently`, the name this package had before the project was renamed, uninstall it and install `controlvar`.
|
|
44
44
|
|
|
45
|
-
The `controlvar` package is on [PyPI](https://pypi.org/project/controlvar/). 0.
|
|
45
|
+
The `controlvar` package is on [PyPI](https://pypi.org/project/controlvar/). 0.3.0 is a
|
|
46
46
|
beta release. It installs and imports as `controlvar`.
|
|
47
47
|
|
|
48
48
|
```bash
|
|
49
|
-
pip install controlvar==0.
|
|
49
|
+
pip install controlvar==0.3.0
|
|
50
50
|
```
|
|
51
51
|
|
|
52
52
|
To work on the SDK itself, install it from the root of a clone of this repository instead:
|
|
@@ -57,19 +57,55 @@ pip install -e sdk/python
|
|
|
57
57
|
|
|
58
58
|
## Changes in this version
|
|
59
59
|
|
|
60
|
-
- `
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
60
|
+
- `get_string_value`, `get_number_value` and `get_json_value` are new: they read the value a
|
|
61
|
+
string, number or json flag serves. Each returns your `default` (keyword-only) when it cannot,
|
|
62
|
+
logs a warning, and passes the reason to `on_error(error, "value")`. See
|
|
63
|
+
[Flag values](https://github.com/Controlvar/controlvar/blob/main/docs/sdk/python.md#flag-values).
|
|
64
|
+
- `FlagEvaluation` gains `value` and `value_type`, after `source`, so building one by position
|
|
65
|
+
still works. `FlagEvaluation` equality, `repr` and `astuple`/`asdict` include the two new fields.
|
|
66
|
+
- `ExperimentationError` gains `code`: `"TYPE_MISMATCH"` or `"INVALID_RESPONSE"` on the errors the
|
|
67
|
+
typed getters make, and `"HOOK_ERROR"` on the error `on_error` receives with `"exposure"`. Other
|
|
68
|
+
errors carry no code in this version; `status` describes them.
|
|
69
|
+
- `on_exposure` is new: a function you give the client when you create it, which it calls once each
|
|
70
|
+
time the server enrols a user in an experiment and the answer did not come from its cache. It
|
|
71
|
+
receives an `Exposure`, which `controlvar` exports. See Exposure hook below.
|
|
72
|
+
- `on_error` is now also called in server mode: with the operation `"value"` when a typed getter
|
|
73
|
+
returns your default, and with `"exposure"` when `on_exposure` raises. Code that treats an
|
|
74
|
+
operation or a `code` it does not know as a mistake should handle them.
|
|
75
|
+
- `track()` takes an `event_id`, and `track_batch()` sends the `event_id` key of an event's dict.
|
|
76
|
+
See Event ids below.
|
|
77
|
+
- `track()` returns `False`, and sends nothing, when `event_value` or `properties` holds NaN or an
|
|
78
|
+
infinite number; see Tracking fan-out rule for the other methods. Before this version the request
|
|
79
|
+
was sent.
|
|
80
|
+
- Local evaluation answers a string, number or json flag's value too. A flag entry with an
|
|
81
|
+
`evaluation` this SDK does not know sends only that flag to the server, and the rest of the
|
|
82
|
+
ruleset is still answered locally; before this version it refused the whole ruleset, and the
|
|
83
|
+
client kept answering from its last one. An entry with a `value_type` it does not know, or a
|
|
84
|
+
value of the wrong type, also sends only that flag to the server.
|
|
85
|
+
- Once local evaluation stops answering a flag (`max_stale_seconds` passed, the ruleset discarded,
|
|
86
|
+
`close()`), a call for a user it answered locally goes to the server; before this version that
|
|
87
|
+
user kept the local answer from the cache for up to `cache_ttl_seconds`.
|
|
88
|
+
- With a finite `cache_ttl_seconds` above 24 hours, a `track()` without a key still reaches an
|
|
89
|
+
experiment or flag whose answer is still cached; before this version it reached only those
|
|
90
|
+
answered in the last 24 hours.
|
|
91
|
+
- The message of a request that fails on the network, and the warning the client logs for it,
|
|
92
|
+
name the URL without its query string, which carries the user id and the user's attributes;
|
|
93
|
+
before this version they named the full URL.
|
|
94
|
+
|
|
95
|
+
## Event ids
|
|
96
|
+
|
|
97
|
+
`track(..., event_id=None)` sends an `event_id`, and `track_batch()` sends the `event_id` key of
|
|
98
|
+
an event's dict as it is. The server stores an event sent again with the same `event_id`, user,
|
|
99
|
+
experiment, flag and event name only once. It needs a Controlvar server that includes the `event_id` change (#1640,
|
|
100
|
+
in the first platform release after it merged): an older server ignores the field.
|
|
101
|
+
|
|
102
|
+
- The SDK makes no id: without an `event_id` none is sent, and an event sent again is counted
|
|
103
|
+
again. Make the id once, when the event happens, and give the same one each time you send it.
|
|
104
|
+
- `None` and `""` send none. The server refuses an id of more than 255 characters or one holding
|
|
105
|
+
NUL with `422`; the SDK does not check it.
|
|
106
|
+
- An event fanned out to several experiments and flags carries the same id in every entry. A `429`
|
|
107
|
+
that is retried sends the same id. A request that fails on the network or times out is not sent
|
|
108
|
+
again by the SDK.
|
|
73
109
|
|
|
74
110
|
## Quick start
|
|
75
111
|
|
|
@@ -96,7 +132,7 @@ assignment.variant_name, assignment.is_control, assignment.configuration
|
|
|
96
132
|
# Feature flags (GET /api/v1/feature-flags/evaluate/{key}?user_id=…&context=<url-encoded attributes>)
|
|
97
133
|
if client.is_feature_enabled("new_search", "user-123", user_attributes={"plan": "pro"}):
|
|
98
134
|
print("use the new search") # your own code
|
|
99
|
-
flag = client.get_feature_flag("new_search", "user-123") # FlagEvaluation(key, enabled, config, reason)
|
|
135
|
+
flag = client.get_feature_flag("new_search", "user-123") # FlagEvaluation(key, enabled, config, reason, source, value, value_type)
|
|
100
136
|
flags = client.get_all_flags("user-123", {"plan": "pro"}) # {"new_search": True, ...}
|
|
101
137
|
|
|
102
138
|
# Tracking (never raises)
|
|
@@ -113,10 +149,13 @@ default_variant="control", transport=None)`. Instances are thread-safe.
|
|
|
113
149
|
|---|---|---|
|
|
114
150
|
| `get_assignment(experiment_key, user_id, user_attributes=None)` | `Assignment(experiment_key, user_id, variant_id, variant_name, is_control, configuration, assigned, reason)` | raises `ExperimentationError` (`status == 404` when no experiment has the key, or for a draft with no control variant); an experiment that is not ACTIVE returns its control variant with `assigned` False and `reason` its status |
|
|
115
151
|
| `get_variant(experiment_key, user_id, user_attributes=None)` | `str` variant name | returns `default_variant` |
|
|
116
|
-
| `get_feature_flag(flag_key, user_id, user_attributes=None)` | `FlagEvaluation(key, enabled, config, reason)` | raises `ExperimentationError` (`status == 404` when the flag key is unknown; a flag that is not ACTIVE returns `enabled=False`, `reason="inactive"`) |
|
|
152
|
+
| `get_feature_flag(flag_key, user_id, user_attributes=None)` | `FlagEvaluation(key, enabled, config, reason, source, value, value_type)` | raises `ExperimentationError` (`status == 404` when the flag key is unknown; a flag that is not ACTIVE returns `enabled=False`, `reason="inactive"`) |
|
|
117
153
|
| `is_feature_enabled(flag_key, user_id, user_attributes=None)` | `bool` | returns `False` |
|
|
154
|
+
| `get_string_value(flag_key, user_id, user_attributes=None, *, default)` | `str`, the string a string flag serves this user | returns `default`; never raises; logs a warning and calls `on_error(error, "value")` |
|
|
155
|
+
| `get_number_value(flag_key, user_id, user_attributes=None, *, default)` | `int` or `float` as saved (never a `bool`), the number a number flag serves this user | returns `default`; never raises; logs a warning and calls `on_error(error, "value")` |
|
|
156
|
+
| `get_json_value(flag_key, user_id, user_attributes=None, *, default)` | a new copy of the `dict` or `list` a json flag serves this user | returns `default`; never raises; logs a warning and calls `on_error(error, "value")` |
|
|
118
157
|
| `get_all_flags(user_id, user_attributes=None)` | `dict[str, bool]` (not cached) | raises `ExperimentationError` |
|
|
119
|
-
| `track(user_id, event_name, event_value=None, properties=None, experiment_key=None, feature_flag_key=None, event_type=None, timestamp=None)` | `bool` — `True` when sent and answered 2xx | never raises; `False` on failure |
|
|
158
|
+
| `track(user_id, event_name, event_value=None, properties=None, experiment_key=None, feature_flag_key=None, event_type=None, timestamp=None, event_id=None)` | `bool` — `True` when sent and answered 2xx | never raises; `False` on failure |
|
|
120
159
|
| `track_batch(events)` | `BatchResult(success_count, failure_count, errors)`; `.ok` | never raises; a failed chunk counts all its events as failures |
|
|
121
160
|
| `report_error(error_type, message, *, feature_flag_key=None, experiment_key=None, user_id=None, stack_trace=None, metadata=None, timestamp=None)` | `bool` — `True` when the server stored the report (201) | never raises; `False` and a logged warning otherwise |
|
|
122
161
|
| `get_assignments(user_id, active_only=True)` | `list[dict]` from the server | raises `ExperimentationError` |
|
|
@@ -125,9 +164,11 @@ default_variant="control", transport=None)`. Instances are thread-safe.
|
|
|
125
164
|
| `ready(timeout_seconds=None)`, `status()`, `close()` / `with` | local mode: readiness, ruleset state, shutdown (sends evaluation counts) | never raise |
|
|
126
165
|
| `consistent_hash(user_id, flag_key)` / `md5_hex(user_id, flag_key)` | cross-SDK MD5 bucket in `[0, 1)` / hex digest | pure functions; **not** the flag bucketing function |
|
|
127
166
|
|
|
128
|
-
`ExperimentationError` carries `.status` (HTTP status, `None` for network errors/timeouts)
|
|
129
|
-
`.body` (raw response text)
|
|
130
|
-
|
|
167
|
+
`ExperimentationError` carries `.status` (HTTP status, `None` for network errors/timeouts),
|
|
168
|
+
`.body` (raw response text) and `.code`: `"TYPE_MISMATCH"` or `"INVALID_RESPONSE"` on an error a
|
|
169
|
+
typed getter makes itself, and `"HOOK_ERROR"` on the error `on_error` receives with `"exposure"`.
|
|
170
|
+
Other errors carry no code in this version; `.status` describes them. Failures are never cached, so
|
|
171
|
+
the next call retries. A `429` is retried once after `Retry-After` seconds (capped at 5 s).
|
|
131
172
|
|
|
132
173
|
`user_attributes` is sent as `context` on assignment **and** on flag evaluation
|
|
133
174
|
(`&context=<url-encoded JSON>`, omitted when empty), so flag targeting rules evaluate against it
|
|
@@ -171,16 +212,78 @@ another process does not have them. In a server, pass `experiment_key` on conver
|
|
|
171
212
|
`event_type` defaults to `event_name`; `properties` is sent as `metadata`; `timestamp` may be a
|
|
172
213
|
`datetime` (naive values are treated as UTC) or an ISO-8601 string.
|
|
173
214
|
|
|
215
|
+
`track` returns `False`, and sends nothing, when `event_value` or `properties` holds NaN or an
|
|
216
|
+
infinite number. `get_assignment` raises `ExperimentationError`
|
|
217
|
+
for the same in `user_attributes`, and `get_variant` returns the default for a user with no
|
|
218
|
+
cached assignment (a cached assignment is returned as before).
|
|
219
|
+
|
|
220
|
+
## Exposure hook
|
|
221
|
+
|
|
222
|
+
`on_exposure` is a function you give the client when you create it. The client calls it once each
|
|
223
|
+
time the server enrols the user in an experiment and the answer did not come from the client's
|
|
224
|
+
cache: the moment the server records the exposure. Use it to send the exposure to your analytics
|
|
225
|
+
tool or warehouse.
|
|
226
|
+
|
|
227
|
+
The example needs an ACTIVE experiment with the key `checkout_flow`; put your own key in its place.
|
|
228
|
+
It prints `exposure checkout_flow <variant> user-123` once, or nothing for a user the experiment
|
|
229
|
+
did not enrol.
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
import os
|
|
233
|
+
import queue
|
|
234
|
+
import threading
|
|
235
|
+
|
|
236
|
+
from controlvar import ExperimentationClient
|
|
237
|
+
|
|
238
|
+
exposures = queue.Queue(maxsize=10_000)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def send_exposures():
|
|
242
|
+
while True:
|
|
243
|
+
exposure = exposures.get()
|
|
244
|
+
# Your analytics tool's call goes here: it runs on this thread, not inside get_variant.
|
|
245
|
+
print("exposure", exposure.experiment_key, exposure.variant_name, exposure.user_id)
|
|
246
|
+
exposures.task_done()
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
threading.Thread(target=send_exposures, daemon=True).start()
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def on_exposure(exposure):
|
|
253
|
+
try:
|
|
254
|
+
exposures.put_nowait(exposure)
|
|
255
|
+
except queue.Full:
|
|
256
|
+
pass # drop it rather than make get_variant wait
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
client = ExperimentationClient(
|
|
260
|
+
api_url=os.environ.get("CONTROLVAR_API_URL", "http://localhost:8000"),
|
|
261
|
+
api_key=os.environ["CONTROLVAR_API_KEY"],
|
|
262
|
+
on_exposure=on_exposure,
|
|
263
|
+
)
|
|
264
|
+
client.get_variant("checkout_flow", "user-123")
|
|
265
|
+
exposures.join()
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
It receives an `Exposure`: `experiment_key`, `user_id`, `variant_id` (`None` when the server sends
|
|
269
|
+
none) and `variant_name`. It is not called for a cached answer, a user the experiment did not enrol
|
|
270
|
+
or an experiment that is not running (`assigned` False), a call that fails, or a feature flag. It
|
|
271
|
+
runs in the thread that called `get_assignment` or `get_variant`, before that call returns, so keep
|
|
272
|
+
it quick. If it raises, the answer is unchanged, and the error is logged at ERROR and passed to
|
|
273
|
+
`on_error` with the operation `"exposure"` and `code` `"HOOK_ERROR"`.
|
|
274
|
+
|
|
275
|
+
Full details: [Exposure hook](https://github.com/Controlvar/controlvar/blob/main/docs/sdk/python.md#exposure-hook).
|
|
276
|
+
|
|
174
277
|
## Backend endpoints used
|
|
175
278
|
|
|
176
279
|
Every request carries `X-API-Key: <key>`, `Content-Type: application/json` and `Accept: application/json`.
|
|
177
280
|
|
|
178
281
|
| SDK call | Method and path | Body / query | Response used |
|
|
179
282
|
|---|---|---|---|
|
|
180
|
-
| `get_assignment`, `get_variant` | `POST /api/v1/tracking/assign` | `{experiment_key, user_id, context?}` | `{experiment_key, user_id, variant_id, variant_name, is_control, configuration}`; 404 when no experiment has the key, or for a draft with no control variant; an experiment that is not ACTIVE answers its control variant with `assigned: false` |
|
|
283
|
+
| `get_assignment`, `get_variant` | `POST /api/v1/tracking/assign` | `{experiment_key, user_id, context?}` | `{experiment_key, user_id, variant_id, variant_name, is_control, configuration, assigned, reason}`; 404 when no experiment has the key, or for a draft with no control variant; an experiment that is not ACTIVE answers its control variant with `assigned: false` |
|
|
181
284
|
| `get_feature_flag`, `is_feature_enabled` | `GET /api/v1/feature-flags/evaluate/{flag_key}?user_id=…&context=<url-encoded JSON>` | `context` = `user_attributes` (omitted when empty) | `{key, enabled, config, reason}`; off with `reason: "inactive"` when the flag exists but is not ACTIVE; 404 only for an unknown key |
|
|
182
285
|
| `get_all_flags` | `GET /api/v1/feature-flags/user/{user_id}?context=<url-encoded JSON>` | `context` = `user_attributes` (omitted when empty) | `{"<flag_key>": bool, …}` |
|
|
183
|
-
| `track` with a key | `POST /api/v1/tracking/track` | `{event_type, event_name, user_id, experiment_key?, feature_flag_key?, value?, metadata?, timestamp?}` | ignored |
|
|
286
|
+
| `track` with a key | `POST /api/v1/tracking/track` | `{event_type, event_name, user_id, experiment_key?, feature_flag_key?, value?, metadata?, timestamp?, event_id?}` | ignored |
|
|
184
287
|
| `track` without keys | `POST /api/v1/tracking/batch`, one entry per experiment and flag it reaches; with none, `POST /api/v1/tracking/track` | `{events: [<track body>, …]}` (≤ 100 per request); with none, the track body with no key | ignored |
|
|
185
288
|
| `track_batch` | `POST /api/v1/tracking/batch` | `{events: [<track body>, …]}` (≤ 100 per request) | `{success_count, failure_count, errors}` |
|
|
186
289
|
| `get_assignments` | `GET /api/v1/tracking/assignments/{user_id}?active_only=true` | — | list of dicts |
|
|
@@ -194,7 +297,7 @@ CONTROLVAR_API_KEY=<key> python sdk/python/examples/contract_smoke.py
|
|
|
194
297
|
It prints:
|
|
195
298
|
|
|
196
299
|
```text
|
|
197
|
-
{"sdk":"python","assign":{"variant_name":"treatment","is_control":false,"sticky":true},"flag":{"enabled":true},"track":{"ok":true},"fanout":{"ok":true}}
|
|
300
|
+
{"sdk":"python","assign":{"variant_name":"treatment","is_control":false,"sticky":true},"flag":{"enabled":true},"track":{"ok":true},"fanout":{"ok":true},"event_id":{"user_id":"smoke-<uuid>","event_name":"contract_dedup_<uuid>","id":"contract-<uuid>","sends":2},"exposure":{"calls":1,"payload":{"experiment_key":"sdk_contract_ab","variant_name":"treatment","variant_id":"<id>","user_id":"smoke-<uuid>"}}}
|
|
198
301
|
```
|
|
199
302
|
|
|
200
303
|
Env: `CONTROLVAR_API_URL` (default `http://localhost:8000`), `CONTROLVAR_API_KEY` (required),
|
|
@@ -16,11 +16,11 @@ else to install.
|
|
|
16
16
|
|
|
17
17
|
If you installed `experimently`, the name this package had before the project was renamed, uninstall it and install `controlvar`.
|
|
18
18
|
|
|
19
|
-
The `controlvar` package is on [PyPI](https://pypi.org/project/controlvar/). 0.
|
|
19
|
+
The `controlvar` package is on [PyPI](https://pypi.org/project/controlvar/). 0.3.0 is a
|
|
20
20
|
beta release. It installs and imports as `controlvar`.
|
|
21
21
|
|
|
22
22
|
```bash
|
|
23
|
-
pip install controlvar==0.
|
|
23
|
+
pip install controlvar==0.3.0
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
To work on the SDK itself, install it from the root of a clone of this repository instead:
|
|
@@ -31,19 +31,55 @@ pip install -e sdk/python
|
|
|
31
31
|
|
|
32
32
|
## Changes in this version
|
|
33
33
|
|
|
34
|
-
- `
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
34
|
+
- `get_string_value`, `get_number_value` and `get_json_value` are new: they read the value a
|
|
35
|
+
string, number or json flag serves. Each returns your `default` (keyword-only) when it cannot,
|
|
36
|
+
logs a warning, and passes the reason to `on_error(error, "value")`. See
|
|
37
|
+
[Flag values](https://github.com/Controlvar/controlvar/blob/main/docs/sdk/python.md#flag-values).
|
|
38
|
+
- `FlagEvaluation` gains `value` and `value_type`, after `source`, so building one by position
|
|
39
|
+
still works. `FlagEvaluation` equality, `repr` and `astuple`/`asdict` include the two new fields.
|
|
40
|
+
- `ExperimentationError` gains `code`: `"TYPE_MISMATCH"` or `"INVALID_RESPONSE"` on the errors the
|
|
41
|
+
typed getters make, and `"HOOK_ERROR"` on the error `on_error` receives with `"exposure"`. Other
|
|
42
|
+
errors carry no code in this version; `status` describes them.
|
|
43
|
+
- `on_exposure` is new: a function you give the client when you create it, which it calls once each
|
|
44
|
+
time the server enrols a user in an experiment and the answer did not come from its cache. It
|
|
45
|
+
receives an `Exposure`, which `controlvar` exports. See Exposure hook below.
|
|
46
|
+
- `on_error` is now also called in server mode: with the operation `"value"` when a typed getter
|
|
47
|
+
returns your default, and with `"exposure"` when `on_exposure` raises. Code that treats an
|
|
48
|
+
operation or a `code` it does not know as a mistake should handle them.
|
|
49
|
+
- `track()` takes an `event_id`, and `track_batch()` sends the `event_id` key of an event's dict.
|
|
50
|
+
See Event ids below.
|
|
51
|
+
- `track()` returns `False`, and sends nothing, when `event_value` or `properties` holds NaN or an
|
|
52
|
+
infinite number; see Tracking fan-out rule for the other methods. Before this version the request
|
|
53
|
+
was sent.
|
|
54
|
+
- Local evaluation answers a string, number or json flag's value too. A flag entry with an
|
|
55
|
+
`evaluation` this SDK does not know sends only that flag to the server, and the rest of the
|
|
56
|
+
ruleset is still answered locally; before this version it refused the whole ruleset, and the
|
|
57
|
+
client kept answering from its last one. An entry with a `value_type` it does not know, or a
|
|
58
|
+
value of the wrong type, also sends only that flag to the server.
|
|
59
|
+
- Once local evaluation stops answering a flag (`max_stale_seconds` passed, the ruleset discarded,
|
|
60
|
+
`close()`), a call for a user it answered locally goes to the server; before this version that
|
|
61
|
+
user kept the local answer from the cache for up to `cache_ttl_seconds`.
|
|
62
|
+
- With a finite `cache_ttl_seconds` above 24 hours, a `track()` without a key still reaches an
|
|
63
|
+
experiment or flag whose answer is still cached; before this version it reached only those
|
|
64
|
+
answered in the last 24 hours.
|
|
65
|
+
- The message of a request that fails on the network, and the warning the client logs for it,
|
|
66
|
+
name the URL without its query string, which carries the user id and the user's attributes;
|
|
67
|
+
before this version they named the full URL.
|
|
68
|
+
|
|
69
|
+
## Event ids
|
|
70
|
+
|
|
71
|
+
`track(..., event_id=None)` sends an `event_id`, and `track_batch()` sends the `event_id` key of
|
|
72
|
+
an event's dict as it is. The server stores an event sent again with the same `event_id`, user,
|
|
73
|
+
experiment, flag and event name only once. It needs a Controlvar server that includes the `event_id` change (#1640,
|
|
74
|
+
in the first platform release after it merged): an older server ignores the field.
|
|
75
|
+
|
|
76
|
+
- The SDK makes no id: without an `event_id` none is sent, and an event sent again is counted
|
|
77
|
+
again. Make the id once, when the event happens, and give the same one each time you send it.
|
|
78
|
+
- `None` and `""` send none. The server refuses an id of more than 255 characters or one holding
|
|
79
|
+
NUL with `422`; the SDK does not check it.
|
|
80
|
+
- An event fanned out to several experiments and flags carries the same id in every entry. A `429`
|
|
81
|
+
that is retried sends the same id. A request that fails on the network or times out is not sent
|
|
82
|
+
again by the SDK.
|
|
47
83
|
|
|
48
84
|
## Quick start
|
|
49
85
|
|
|
@@ -70,7 +106,7 @@ assignment.variant_name, assignment.is_control, assignment.configuration
|
|
|
70
106
|
# Feature flags (GET /api/v1/feature-flags/evaluate/{key}?user_id=…&context=<url-encoded attributes>)
|
|
71
107
|
if client.is_feature_enabled("new_search", "user-123", user_attributes={"plan": "pro"}):
|
|
72
108
|
print("use the new search") # your own code
|
|
73
|
-
flag = client.get_feature_flag("new_search", "user-123") # FlagEvaluation(key, enabled, config, reason)
|
|
109
|
+
flag = client.get_feature_flag("new_search", "user-123") # FlagEvaluation(key, enabled, config, reason, source, value, value_type)
|
|
74
110
|
flags = client.get_all_flags("user-123", {"plan": "pro"}) # {"new_search": True, ...}
|
|
75
111
|
|
|
76
112
|
# Tracking (never raises)
|
|
@@ -87,10 +123,13 @@ default_variant="control", transport=None)`. Instances are thread-safe.
|
|
|
87
123
|
|---|---|---|
|
|
88
124
|
| `get_assignment(experiment_key, user_id, user_attributes=None)` | `Assignment(experiment_key, user_id, variant_id, variant_name, is_control, configuration, assigned, reason)` | raises `ExperimentationError` (`status == 404` when no experiment has the key, or for a draft with no control variant); an experiment that is not ACTIVE returns its control variant with `assigned` False and `reason` its status |
|
|
89
125
|
| `get_variant(experiment_key, user_id, user_attributes=None)` | `str` variant name | returns `default_variant` |
|
|
90
|
-
| `get_feature_flag(flag_key, user_id, user_attributes=None)` | `FlagEvaluation(key, enabled, config, reason)` | raises `ExperimentationError` (`status == 404` when the flag key is unknown; a flag that is not ACTIVE returns `enabled=False`, `reason="inactive"`) |
|
|
126
|
+
| `get_feature_flag(flag_key, user_id, user_attributes=None)` | `FlagEvaluation(key, enabled, config, reason, source, value, value_type)` | raises `ExperimentationError` (`status == 404` when the flag key is unknown; a flag that is not ACTIVE returns `enabled=False`, `reason="inactive"`) |
|
|
91
127
|
| `is_feature_enabled(flag_key, user_id, user_attributes=None)` | `bool` | returns `False` |
|
|
128
|
+
| `get_string_value(flag_key, user_id, user_attributes=None, *, default)` | `str`, the string a string flag serves this user | returns `default`; never raises; logs a warning and calls `on_error(error, "value")` |
|
|
129
|
+
| `get_number_value(flag_key, user_id, user_attributes=None, *, default)` | `int` or `float` as saved (never a `bool`), the number a number flag serves this user | returns `default`; never raises; logs a warning and calls `on_error(error, "value")` |
|
|
130
|
+
| `get_json_value(flag_key, user_id, user_attributes=None, *, default)` | a new copy of the `dict` or `list` a json flag serves this user | returns `default`; never raises; logs a warning and calls `on_error(error, "value")` |
|
|
92
131
|
| `get_all_flags(user_id, user_attributes=None)` | `dict[str, bool]` (not cached) | raises `ExperimentationError` |
|
|
93
|
-
| `track(user_id, event_name, event_value=None, properties=None, experiment_key=None, feature_flag_key=None, event_type=None, timestamp=None)` | `bool` — `True` when sent and answered 2xx | never raises; `False` on failure |
|
|
132
|
+
| `track(user_id, event_name, event_value=None, properties=None, experiment_key=None, feature_flag_key=None, event_type=None, timestamp=None, event_id=None)` | `bool` — `True` when sent and answered 2xx | never raises; `False` on failure |
|
|
94
133
|
| `track_batch(events)` | `BatchResult(success_count, failure_count, errors)`; `.ok` | never raises; a failed chunk counts all its events as failures |
|
|
95
134
|
| `report_error(error_type, message, *, feature_flag_key=None, experiment_key=None, user_id=None, stack_trace=None, metadata=None, timestamp=None)` | `bool` — `True` when the server stored the report (201) | never raises; `False` and a logged warning otherwise |
|
|
96
135
|
| `get_assignments(user_id, active_only=True)` | `list[dict]` from the server | raises `ExperimentationError` |
|
|
@@ -99,9 +138,11 @@ default_variant="control", transport=None)`. Instances are thread-safe.
|
|
|
99
138
|
| `ready(timeout_seconds=None)`, `status()`, `close()` / `with` | local mode: readiness, ruleset state, shutdown (sends evaluation counts) | never raise |
|
|
100
139
|
| `consistent_hash(user_id, flag_key)` / `md5_hex(user_id, flag_key)` | cross-SDK MD5 bucket in `[0, 1)` / hex digest | pure functions; **not** the flag bucketing function |
|
|
101
140
|
|
|
102
|
-
`ExperimentationError` carries `.status` (HTTP status, `None` for network errors/timeouts)
|
|
103
|
-
`.body` (raw response text)
|
|
104
|
-
|
|
141
|
+
`ExperimentationError` carries `.status` (HTTP status, `None` for network errors/timeouts),
|
|
142
|
+
`.body` (raw response text) and `.code`: `"TYPE_MISMATCH"` or `"INVALID_RESPONSE"` on an error a
|
|
143
|
+
typed getter makes itself, and `"HOOK_ERROR"` on the error `on_error` receives with `"exposure"`.
|
|
144
|
+
Other errors carry no code in this version; `.status` describes them. Failures are never cached, so
|
|
145
|
+
the next call retries. A `429` is retried once after `Retry-After` seconds (capped at 5 s).
|
|
105
146
|
|
|
106
147
|
`user_attributes` is sent as `context` on assignment **and** on flag evaluation
|
|
107
148
|
(`&context=<url-encoded JSON>`, omitted when empty), so flag targeting rules evaluate against it
|
|
@@ -145,16 +186,78 @@ another process does not have them. In a server, pass `experiment_key` on conver
|
|
|
145
186
|
`event_type` defaults to `event_name`; `properties` is sent as `metadata`; `timestamp` may be a
|
|
146
187
|
`datetime` (naive values are treated as UTC) or an ISO-8601 string.
|
|
147
188
|
|
|
189
|
+
`track` returns `False`, and sends nothing, when `event_value` or `properties` holds NaN or an
|
|
190
|
+
infinite number. `get_assignment` raises `ExperimentationError`
|
|
191
|
+
for the same in `user_attributes`, and `get_variant` returns the default for a user with no
|
|
192
|
+
cached assignment (a cached assignment is returned as before).
|
|
193
|
+
|
|
194
|
+
## Exposure hook
|
|
195
|
+
|
|
196
|
+
`on_exposure` is a function you give the client when you create it. The client calls it once each
|
|
197
|
+
time the server enrols the user in an experiment and the answer did not come from the client's
|
|
198
|
+
cache: the moment the server records the exposure. Use it to send the exposure to your analytics
|
|
199
|
+
tool or warehouse.
|
|
200
|
+
|
|
201
|
+
The example needs an ACTIVE experiment with the key `checkout_flow`; put your own key in its place.
|
|
202
|
+
It prints `exposure checkout_flow <variant> user-123` once, or nothing for a user the experiment
|
|
203
|
+
did not enrol.
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
import os
|
|
207
|
+
import queue
|
|
208
|
+
import threading
|
|
209
|
+
|
|
210
|
+
from controlvar import ExperimentationClient
|
|
211
|
+
|
|
212
|
+
exposures = queue.Queue(maxsize=10_000)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def send_exposures():
|
|
216
|
+
while True:
|
|
217
|
+
exposure = exposures.get()
|
|
218
|
+
# Your analytics tool's call goes here: it runs on this thread, not inside get_variant.
|
|
219
|
+
print("exposure", exposure.experiment_key, exposure.variant_name, exposure.user_id)
|
|
220
|
+
exposures.task_done()
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
threading.Thread(target=send_exposures, daemon=True).start()
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def on_exposure(exposure):
|
|
227
|
+
try:
|
|
228
|
+
exposures.put_nowait(exposure)
|
|
229
|
+
except queue.Full:
|
|
230
|
+
pass # drop it rather than make get_variant wait
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
client = ExperimentationClient(
|
|
234
|
+
api_url=os.environ.get("CONTROLVAR_API_URL", "http://localhost:8000"),
|
|
235
|
+
api_key=os.environ["CONTROLVAR_API_KEY"],
|
|
236
|
+
on_exposure=on_exposure,
|
|
237
|
+
)
|
|
238
|
+
client.get_variant("checkout_flow", "user-123")
|
|
239
|
+
exposures.join()
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
It receives an `Exposure`: `experiment_key`, `user_id`, `variant_id` (`None` when the server sends
|
|
243
|
+
none) and `variant_name`. It is not called for a cached answer, a user the experiment did not enrol
|
|
244
|
+
or an experiment that is not running (`assigned` False), a call that fails, or a feature flag. It
|
|
245
|
+
runs in the thread that called `get_assignment` or `get_variant`, before that call returns, so keep
|
|
246
|
+
it quick. If it raises, the answer is unchanged, and the error is logged at ERROR and passed to
|
|
247
|
+
`on_error` with the operation `"exposure"` and `code` `"HOOK_ERROR"`.
|
|
248
|
+
|
|
249
|
+
Full details: [Exposure hook](https://github.com/Controlvar/controlvar/blob/main/docs/sdk/python.md#exposure-hook).
|
|
250
|
+
|
|
148
251
|
## Backend endpoints used
|
|
149
252
|
|
|
150
253
|
Every request carries `X-API-Key: <key>`, `Content-Type: application/json` and `Accept: application/json`.
|
|
151
254
|
|
|
152
255
|
| SDK call | Method and path | Body / query | Response used |
|
|
153
256
|
|---|---|---|---|
|
|
154
|
-
| `get_assignment`, `get_variant` | `POST /api/v1/tracking/assign` | `{experiment_key, user_id, context?}` | `{experiment_key, user_id, variant_id, variant_name, is_control, configuration}`; 404 when no experiment has the key, or for a draft with no control variant; an experiment that is not ACTIVE answers its control variant with `assigned: false` |
|
|
257
|
+
| `get_assignment`, `get_variant` | `POST /api/v1/tracking/assign` | `{experiment_key, user_id, context?}` | `{experiment_key, user_id, variant_id, variant_name, is_control, configuration, assigned, reason}`; 404 when no experiment has the key, or for a draft with no control variant; an experiment that is not ACTIVE answers its control variant with `assigned: false` |
|
|
155
258
|
| `get_feature_flag`, `is_feature_enabled` | `GET /api/v1/feature-flags/evaluate/{flag_key}?user_id=…&context=<url-encoded JSON>` | `context` = `user_attributes` (omitted when empty) | `{key, enabled, config, reason}`; off with `reason: "inactive"` when the flag exists but is not ACTIVE; 404 only for an unknown key |
|
|
156
259
|
| `get_all_flags` | `GET /api/v1/feature-flags/user/{user_id}?context=<url-encoded JSON>` | `context` = `user_attributes` (omitted when empty) | `{"<flag_key>": bool, …}` |
|
|
157
|
-
| `track` with a key | `POST /api/v1/tracking/track` | `{event_type, event_name, user_id, experiment_key?, feature_flag_key?, value?, metadata?, timestamp?}` | ignored |
|
|
260
|
+
| `track` with a key | `POST /api/v1/tracking/track` | `{event_type, event_name, user_id, experiment_key?, feature_flag_key?, value?, metadata?, timestamp?, event_id?}` | ignored |
|
|
158
261
|
| `track` without keys | `POST /api/v1/tracking/batch`, one entry per experiment and flag it reaches; with none, `POST /api/v1/tracking/track` | `{events: [<track body>, …]}` (≤ 100 per request); with none, the track body with no key | ignored |
|
|
159
262
|
| `track_batch` | `POST /api/v1/tracking/batch` | `{events: [<track body>, …]}` (≤ 100 per request) | `{success_count, failure_count, errors}` |
|
|
160
263
|
| `get_assignments` | `GET /api/v1/tracking/assignments/{user_id}?active_only=true` | — | list of dicts |
|
|
@@ -168,7 +271,7 @@ CONTROLVAR_API_KEY=<key> python sdk/python/examples/contract_smoke.py
|
|
|
168
271
|
It prints:
|
|
169
272
|
|
|
170
273
|
```text
|
|
171
|
-
{"sdk":"python","assign":{"variant_name":"treatment","is_control":false,"sticky":true},"flag":{"enabled":true},"track":{"ok":true},"fanout":{"ok":true}}
|
|
274
|
+
{"sdk":"python","assign":{"variant_name":"treatment","is_control":false,"sticky":true},"flag":{"enabled":true},"track":{"ok":true},"fanout":{"ok":true},"event_id":{"user_id":"smoke-<uuid>","event_name":"contract_dedup_<uuid>","id":"contract-<uuid>","sends":2},"exposure":{"calls":1,"payload":{"experiment_key":"sdk_contract_ab","variant_name":"treatment","variant_id":"<id>","user_id":"smoke-<uuid>"}}}
|
|
172
275
|
```
|
|
173
276
|
|
|
174
277
|
Env: `CONTROLVAR_API_URL` (default `http://localhost:8000`), `CONTROLVAR_API_KEY` (required),
|
|
@@ -15,7 +15,13 @@ from .client import BATCH_LIMIT, ExperimentationClient
|
|
|
15
15
|
from .hashing import consistent_hash, md5_hex
|
|
16
16
|
from .local import LocalEvaluationStatus, ReadyResult
|
|
17
17
|
from .transport import Response, Transport, TransportError, UrllibTransport
|
|
18
|
-
from .types import
|
|
18
|
+
from .types import (
|
|
19
|
+
Assignment,
|
|
20
|
+
BatchResult,
|
|
21
|
+
ExperimentationError,
|
|
22
|
+
Exposure,
|
|
23
|
+
FlagEvaluation,
|
|
24
|
+
)
|
|
19
25
|
from .version import __version__
|
|
20
26
|
|
|
21
27
|
__all__ = [
|
|
@@ -24,6 +30,7 @@ __all__ = [
|
|
|
24
30
|
"BatchResult",
|
|
25
31
|
"ExperimentationClient",
|
|
26
32
|
"ExperimentationError",
|
|
33
|
+
"Exposure",
|
|
27
34
|
"FlagEvaluation",
|
|
28
35
|
"LocalEvaluationStatus",
|
|
29
36
|
"ReadyResult",
|
|
@@ -41,7 +41,10 @@ class AttributedKeys:
|
|
|
41
41
|
patches it moves this record and the cache together.
|
|
42
42
|
"""
|
|
43
43
|
|
|
44
|
-
def __init__(self) -> None:
|
|
44
|
+
def __init__(self, cache_ttl_seconds: float = 0) -> None:
|
|
45
|
+
# 24 hours, or the cache lifetime when that is longer, so a cached answer never
|
|
46
|
+
# outlives the record that tags its events (#1423).
|
|
47
|
+
self._horizon = max(ATTRIBUTION_HORIZON_SECONDS, cache_ttl_seconds)
|
|
45
48
|
self._lock = threading.Lock()
|
|
46
49
|
self._users: "OrderedDict[str, Tuple[_Answers, _Answers]]" = OrderedDict()
|
|
47
50
|
|
|
@@ -72,7 +75,7 @@ class AttributedKeys:
|
|
|
72
75
|
if answers is None:
|
|
73
76
|
return [], []
|
|
74
77
|
now = cache._now()
|
|
75
|
-
experiments, flags = _live(answers[0], now), _live(answers[1], now)
|
|
78
|
+
experiments, flags = _live(answers[0], now, self._horizon), _live(answers[1], now, self._horizon)
|
|
76
79
|
if not answers[0] and not answers[1]:
|
|
77
80
|
del self._users[user_id]
|
|
78
81
|
return experiments, flags
|
|
@@ -95,10 +98,10 @@ class AttributedKeys:
|
|
|
95
98
|
self._users.popitem(last=False)
|
|
96
99
|
|
|
97
100
|
|
|
98
|
-
def _live(answers: _Answers, now: float) -> List[str]:
|
|
101
|
+
def _live(answers: _Answers, now: float, horizon: float) -> List[str]:
|
|
99
102
|
out: List[str] = []
|
|
100
103
|
for key, (answered_at, reason) in list(answers.items()):
|
|
101
|
-
if now - answered_at >
|
|
104
|
+
if now - answered_at > horizon:
|
|
102
105
|
del answers[key]
|
|
103
106
|
elif reason not in LIFECYCLE_REASONS:
|
|
104
107
|
out.append(key)
|
|
@@ -38,6 +38,20 @@ class UserKeyCache(Generic[T]):
|
|
|
38
38
|
with self._lock:
|
|
39
39
|
self._users.setdefault(user_id, {})[key] = (_now() + self._ttl, value)
|
|
40
40
|
|
|
41
|
+
def set_new(self, user_id: str, key: str, value: T) -> bool:
|
|
42
|
+
"""Store ``value`` as :meth:`set` does, and say whether it is new: ``True`` when the slot
|
|
43
|
+
held no unexpired value (it was empty or expired), ``False`` when it replaced a live one.
|
|
44
|
+
|
|
45
|
+
Checked and stored under the one lock: of two threads storing the same user and key, the
|
|
46
|
+
second finds the first one's value, and sees ``False`` while that value is unexpired."""
|
|
47
|
+
with self._lock:
|
|
48
|
+
by_key = self._users.setdefault(user_id, {})
|
|
49
|
+
now = _now()
|
|
50
|
+
entry = by_key.get(key)
|
|
51
|
+
new = entry is None or now >= entry[0]
|
|
52
|
+
by_key[key] = (now + self._ttl, value)
|
|
53
|
+
return new
|
|
54
|
+
|
|
41
55
|
def entries(self, user_id: str) -> List[Tuple[str, T]]:
|
|
42
56
|
"""All live (unexpired) entries for a user, in insertion order."""
|
|
43
57
|
with self._lock:
|