arcane-flask 2.2.2__tar.gz → 3.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/PKG-INFO +96 -1
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/README.md +95 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/authentication.py +109 -97
- arcane_flask-3.1.0/arcane/flask/tracking/__init__.py +19 -0
- arcane_flask-3.1.0/arcane/flask/tracking/adapters/__init__.py +10 -0
- arcane_flask-3.1.0/arcane/flask/tracking/adapters/log.py +31 -0
- arcane_flask-3.1.0/arcane/flask/tracking/adapters/pubsub.py +65 -0
- arcane_flask-3.1.0/arcane/flask/tracking/context.py +40 -0
- arcane_flask-3.1.0/arcane/flask/tracking/events.py +24 -0
- arcane_flask-3.1.0/arcane/flask/tracking/flask_integration.py +65 -0
- arcane_flask-3.1.0/arcane/flask/tracking/ports.py +14 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/pyproject.toml +1 -1
- arcane_flask-2.2.2/arcane/flask/tracking.py +0 -49
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/__init__.py +0 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/exception_decorator.py +0 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/http_response.py +0 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/log.py +0 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/services.py +0 -0
- {arcane_flask-2.2.2 → arcane_flask-3.1.0}/arcane/flask/types.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arcane-flask
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.1.0
|
|
4
4
|
Summary: Utility functions for flask apps.
|
|
5
5
|
Author: Arcane
|
|
6
6
|
Author-email: product@wearcane.com
|
|
@@ -88,6 +88,101 @@ If the function exceeds the timeout, the decorator will return:
|
|
|
88
88
|
|
|
89
89
|
**Note:** Set the `timeout` parameter to a value a few seconds less than your Cloud Run or server timeout configuration to ensure proper error handling.
|
|
90
90
|
|
|
91
|
+
## Activity tracking
|
|
92
|
+
|
|
93
|
+
Tracking records request usage (user, service, function, status code, execution time, optional metadata) and sends it to a configurable sink via a **ports & adapters** setup. You pass a **tracker** (adapter) and a **service name** at startup; every request is then tracked automatically.
|
|
94
|
+
|
|
95
|
+
### Register tracking at app startup
|
|
96
|
+
|
|
97
|
+
Use `init_tracking(app, tracker, service_name)`. The tracker is any implementation of `TrackingPort` (e.g. `LogTrackingAdapter` or `PubSubTrackingAdapter`).
|
|
98
|
+
|
|
99
|
+
**Log adapter** (development or debugging; events go to the Python logger at INFO):
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from arcane.flask import init_tracking
|
|
103
|
+
from arcane.flask.tracking import LogTrackingAdapter
|
|
104
|
+
|
|
105
|
+
app = Flask(__name__)
|
|
106
|
+
tracker = LogTrackingAdapter()
|
|
107
|
+
init_tracking(app, tracker, service_name="my-service")
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**PubSub adapter** (production; events go to a Google Cloud Pub/Sub topic):
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from arcane.flask import init_tracking
|
|
114
|
+
from arcane.flask.tracking import PubSubTrackingAdapter
|
|
115
|
+
from arcane.pubsub import Client as PubSubClient
|
|
116
|
+
|
|
117
|
+
app = Flask(__name__)
|
|
118
|
+
pubsub_client = PubSubClient()
|
|
119
|
+
tracker = PubSubTrackingAdapter(
|
|
120
|
+
pubsub_client=pubsub_client,
|
|
121
|
+
project="my-project",
|
|
122
|
+
topic="activity-tracker", # optional; default is "activity-tracker"
|
|
123
|
+
)
|
|
124
|
+
init_tracking(app, tracker, service_name="my-service")
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
**Custom columns (PubSub only)**
|
|
128
|
+
By default the PubSub adapter sends all standard fields. You can restrict which fields are published by passing `columns`:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
tracker = PubSubTrackingAdapter(
|
|
132
|
+
pubsub_client=pubsub_client,
|
|
133
|
+
project="my-project",
|
|
134
|
+
topic="activity-tracker",
|
|
135
|
+
columns=["user_email", "service", "status_code", "execution_time"],
|
|
136
|
+
)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
### Undecorated routes are tracked
|
|
140
|
+
|
|
141
|
+
Routes that do **not** use `check_access_rights` are still tracked. The middleware fills in a baseline event using the request: `function_name` comes from Flask’s `request.endpoint` (the view function name), `service` from the `service_name` you passed to `init_tracking`, and `email` is empty.
|
|
142
|
+
|
|
143
|
+
### Enriching context during the request
|
|
144
|
+
|
|
145
|
+
Use `enrich_tracking()` inside any route (with or without `check_access_rights`) to add or merge key/value pairs into the event’s `additional_info`. Multiple calls in the same request are merged.
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from arcane.flask.tracking import enrich_tracking
|
|
149
|
+
|
|
150
|
+
@app.route("/process")
|
|
151
|
+
def process():
|
|
152
|
+
enrich_tracking({"labels": ["feed"]})
|
|
153
|
+
size = read_file_size()
|
|
154
|
+
enrich_tracking({"file_size": size})
|
|
155
|
+
return {"status": "ok"}, 200
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
The emitted event will include `additional_info={"labels": ["feed"], "file_size": ...}`.
|
|
159
|
+
|
|
160
|
+
### Excluding endpoints
|
|
161
|
+
|
|
162
|
+
Pass `excluded_endpoints` so some views (e.g. health checks) are not tracked:
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
init_tracking(
|
|
166
|
+
app,
|
|
167
|
+
tracker,
|
|
168
|
+
service_name="my-service",
|
|
169
|
+
excluded_endpoints=frozenset({"health", "readiness"}),
|
|
170
|
+
)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
### Custom adapter
|
|
174
|
+
|
|
175
|
+
Implement the `TrackingPort` interface and pass it to `init_tracking`:
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
from arcane.flask.tracking import TrackingPort, ActivityEvent
|
|
179
|
+
|
|
180
|
+
class MyTrackingAdapter(TrackingPort):
|
|
181
|
+
def emit(self, event: ActivityEvent) -> None:
|
|
182
|
+
# send event to your backend (Kafka, HTTP, etc.)
|
|
183
|
+
...
|
|
184
|
+
```
|
|
185
|
+
|
|
91
186
|
## Structured logging labels
|
|
92
187
|
|
|
93
188
|
The `arcane.flask.logs` module lets you add structured labels to every log
|
|
@@ -62,6 +62,101 @@ If the function exceeds the timeout, the decorator will return:
|
|
|
62
62
|
|
|
63
63
|
**Note:** Set the `timeout` parameter to a value a few seconds less than your Cloud Run or server timeout configuration to ensure proper error handling.
|
|
64
64
|
|
|
65
|
+
## Activity tracking
|
|
66
|
+
|
|
67
|
+
Tracking records request usage (user, service, function, status code, execution time, optional metadata) and sends it to a configurable sink via a **ports & adapters** setup. You pass a **tracker** (adapter) and a **service name** at startup; every request is then tracked automatically.
|
|
68
|
+
|
|
69
|
+
### Register tracking at app startup
|
|
70
|
+
|
|
71
|
+
Use `init_tracking(app, tracker, service_name)`. The tracker is any implementation of `TrackingPort` (e.g. `LogTrackingAdapter` or `PubSubTrackingAdapter`).
|
|
72
|
+
|
|
73
|
+
**Log adapter** (development or debugging; events go to the Python logger at INFO):
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from arcane.flask import init_tracking
|
|
77
|
+
from arcane.flask.tracking import LogTrackingAdapter
|
|
78
|
+
|
|
79
|
+
app = Flask(__name__)
|
|
80
|
+
tracker = LogTrackingAdapter()
|
|
81
|
+
init_tracking(app, tracker, service_name="my-service")
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**PubSub adapter** (production; events go to a Google Cloud Pub/Sub topic):
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from arcane.flask import init_tracking
|
|
88
|
+
from arcane.flask.tracking import PubSubTrackingAdapter
|
|
89
|
+
from arcane.pubsub import Client as PubSubClient
|
|
90
|
+
|
|
91
|
+
app = Flask(__name__)
|
|
92
|
+
pubsub_client = PubSubClient()
|
|
93
|
+
tracker = PubSubTrackingAdapter(
|
|
94
|
+
pubsub_client=pubsub_client,
|
|
95
|
+
project="my-project",
|
|
96
|
+
topic="activity-tracker", # optional; default is "activity-tracker"
|
|
97
|
+
)
|
|
98
|
+
init_tracking(app, tracker, service_name="my-service")
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Custom columns (PubSub only)**
|
|
102
|
+
By default the PubSub adapter sends all standard fields. You can restrict which fields are published by passing `columns`:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
tracker = PubSubTrackingAdapter(
|
|
106
|
+
pubsub_client=pubsub_client,
|
|
107
|
+
project="my-project",
|
|
108
|
+
topic="activity-tracker",
|
|
109
|
+
columns=["user_email", "service", "status_code", "execution_time"],
|
|
110
|
+
)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### Undecorated routes are tracked
|
|
114
|
+
|
|
115
|
+
Routes that do **not** use `check_access_rights` are still tracked. The middleware fills in a baseline event using the request: `function_name` comes from Flask’s `request.endpoint` (the view function name), `service` from the `service_name` you passed to `init_tracking`, and `email` is empty.
|
|
116
|
+
|
|
117
|
+
### Enriching context during the request
|
|
118
|
+
|
|
119
|
+
Use `enrich_tracking()` inside any route (with or without `check_access_rights`) to add or merge key/value pairs into the event’s `additional_info`. Multiple calls in the same request are merged.
|
|
120
|
+
|
|
121
|
+
```python
|
|
122
|
+
from arcane.flask.tracking import enrich_tracking
|
|
123
|
+
|
|
124
|
+
@app.route("/process")
|
|
125
|
+
def process():
|
|
126
|
+
enrich_tracking({"labels": ["feed"]})
|
|
127
|
+
size = read_file_size()
|
|
128
|
+
enrich_tracking({"file_size": size})
|
|
129
|
+
return {"status": "ok"}, 200
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The emitted event will include `additional_info={"labels": ["feed"], "file_size": ...}`.
|
|
133
|
+
|
|
134
|
+
### Excluding endpoints
|
|
135
|
+
|
|
136
|
+
Pass `excluded_endpoints` so some views (e.g. health checks) are not tracked:
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
init_tracking(
|
|
140
|
+
app,
|
|
141
|
+
tracker,
|
|
142
|
+
service_name="my-service",
|
|
143
|
+
excluded_endpoints=frozenset({"health", "readiness"}),
|
|
144
|
+
)
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### Custom adapter
|
|
148
|
+
|
|
149
|
+
Implement the `TrackingPort` interface and pass it to `init_tracking`:
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from arcane.flask.tracking import TrackingPort, ActivityEvent
|
|
153
|
+
|
|
154
|
+
class MyTrackingAdapter(TrackingPort):
|
|
155
|
+
def emit(self, event: ActivityEvent) -> None:
|
|
156
|
+
# send event to your backend (Kafka, HTTP, etc.)
|
|
157
|
+
...
|
|
158
|
+
```
|
|
159
|
+
|
|
65
160
|
## Structured logging labels
|
|
66
161
|
|
|
67
162
|
The `arcane.flask.logs` module lets you add structured labels to every log
|
|
@@ -1,22 +1,20 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
import functools
|
|
2
3
|
import inspect
|
|
3
|
-
import time
|
|
4
4
|
import json
|
|
5
5
|
from typing import Callable, Optional, Union, Dict
|
|
6
6
|
import logging
|
|
7
7
|
|
|
8
8
|
import backoff
|
|
9
9
|
from firebase_admin import auth as firebase_auth
|
|
10
|
-
from flask import request
|
|
10
|
+
from flask import g, request
|
|
11
11
|
from pydash import get
|
|
12
|
-
from google.cloud.exceptions import ServiceUnavailable
|
|
13
12
|
|
|
14
13
|
from arcane.core import CLIENTS_PARAM, ALL_CLIENTS_RIGHTS, RightsLevelEnum, GOOGLE_EXCEPTIONS_TO_RETRY, BadRequestError
|
|
15
14
|
from arcane.datastore import Client as DatastoreClient
|
|
16
15
|
from arcane.pubsub import Client as PubSubClient
|
|
17
|
-
import concurrent.futures
|
|
18
16
|
|
|
19
|
-
from .tracking import
|
|
17
|
+
from arcane.flask.tracking.context import tracking_ctx, TrackingContext
|
|
20
18
|
|
|
21
19
|
CLIENTS_KIND = 'base-clients'
|
|
22
20
|
USERS_KIND = 'users'
|
|
@@ -72,109 +70,123 @@ def check_access_rights(service: str,
|
|
|
72
70
|
|
|
73
71
|
@functools.wraps(job_func)
|
|
74
72
|
def wrapper(*args, **kwargs):
|
|
75
|
-
start = time.time()
|
|
76
73
|
is_appengine_cron = request.headers.get('X-Appengine-Cron', False)
|
|
77
74
|
|
|
78
|
-
if is_appengine_cron:
|
|
79
|
-
logging.info("App Engine calls service : " + service)
|
|
80
|
-
return job_func(*args, **kwargs)
|
|
81
|
-
|
|
82
75
|
claims = {}
|
|
83
76
|
origin = ''
|
|
84
|
-
|
|
85
|
-
if auth_enabled:
|
|
86
|
-
token = str(request.headers.get('Authorization', '')).split(' ').pop()
|
|
87
|
-
origin = str(request.headers.get('origin', ''))
|
|
88
|
-
try:
|
|
89
|
-
claims = verify_token(token)
|
|
90
|
-
except (firebase_auth.ExpiredIdTokenError,
|
|
91
|
-
firebase_auth.RevokedIdTokenError,
|
|
92
|
-
firebase_auth.InvalidIdTokenError) as e:
|
|
93
|
-
if required_rights != RightsLevelEnum.NONE:
|
|
94
|
-
return {'detail': 'You don\'t have access to Adscale resources\n', 'firebase_log': str(e)}, 401
|
|
95
|
-
except ValueError as e:
|
|
96
|
-
return {'detail': str(e)}, 401
|
|
77
|
+
_inner_additional_info = [None]
|
|
97
78
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
79
|
+
async def async_execution():
|
|
80
|
+
nonlocal claims, origin
|
|
81
|
+
|
|
82
|
+
if is_appengine_cron:
|
|
83
|
+
logging.info("App Engine calls service : " + service)
|
|
84
|
+
if asyncio.iscoroutinefunction(job_func):
|
|
85
|
+
return await job_func(*args, **kwargs)
|
|
86
|
+
else:
|
|
87
|
+
return job_func(*args, **kwargs)
|
|
88
|
+
|
|
89
|
+
# Check feature rights
|
|
90
|
+
if auth_enabled:
|
|
91
|
+
token = str(request.headers.get('Authorization', '')).split(' ').pop()
|
|
92
|
+
origin = str(request.headers.get('origin', ''))
|
|
93
|
+
try:
|
|
94
|
+
claims = verify_token(token)
|
|
95
|
+
except (firebase_auth.ExpiredIdTokenError,
|
|
96
|
+
firebase_auth.RevokedIdTokenError,
|
|
97
|
+
firebase_auth.InvalidIdTokenError) as e:
|
|
98
|
+
if required_rights != RightsLevelEnum.NONE:
|
|
99
|
+
return {'detail': 'You don\'t have access to Adscale resources\n', 'firebase_log': str(e)}, 401
|
|
100
|
+
except ValueError as e:
|
|
101
|
+
return {'detail': str(e)}, 401
|
|
102
|
+
|
|
103
|
+
try:
|
|
104
|
+
_check_feature_right_access(
|
|
105
|
+
claims,
|
|
106
|
+
required_rights,
|
|
107
|
+
service_user_right
|
|
108
|
+
)
|
|
109
|
+
except BadRequestError:
|
|
110
|
+
rights = _get_level_access_for_a_feature_from_claims(claims, service_user_right)
|
|
111
|
+
return {'detail': f"You don't have access to this resource : your rights are "
|
|
112
|
+
f"'{rights_mapper.get(str(rights), '0')}' for service '{service}'. "
|
|
113
|
+
f"You need '{rights_mapper.get(str(required_rights), '0')}'.",
|
|
114
|
+
'claims': f"Your current rights are {json.dumps(claims, indent=4, sort_keys=True)}"}, 401
|
|
115
|
+
|
|
116
|
+
# Fill authorized_clients
|
|
117
|
+
if receive_rights_per_client:
|
|
118
|
+
if not auth_enabled:
|
|
119
|
+
authorized_clients = [ALL_CLIENTS_RIGHTS]
|
|
121
120
|
else:
|
|
122
|
-
|
|
123
|
-
|
|
121
|
+
# Value defined for service to service calls
|
|
122
|
+
service_authorized_clients = claims.get('authorized_clients')
|
|
123
|
+
db_user_id = claims.get('db_user_id')
|
|
124
|
+
if service_authorized_clients and not db_user_id:
|
|
125
|
+
authorized_clients = service_authorized_clients
|
|
124
126
|
else:
|
|
125
|
-
|
|
126
|
-
if user is None:
|
|
127
|
+
if not db_user_id:
|
|
127
128
|
authorized_clients = []
|
|
128
129
|
else:
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
except concurrent.futures.TimeoutError:
|
|
144
|
-
func_response = {'detail': 'Request Timeout'}, 504
|
|
145
|
-
except Exception as e:
|
|
146
|
-
func_response = {'detail': f'Server Error: {str(e)}'}, 500
|
|
147
|
-
|
|
148
|
-
end = time.time()
|
|
149
|
-
|
|
150
|
-
additionnal_info = {}
|
|
151
|
-
for key, path in kwargs_name_and_path_to_get.items():
|
|
152
|
-
additionnal_info[key] = get(kwargs, path)
|
|
153
|
-
if len(additionnal_info.keys()) == 0:
|
|
154
|
-
additionnal_info = None
|
|
155
|
-
|
|
156
|
-
if auth_enabled:
|
|
157
|
-
status_code = 200
|
|
158
|
-
if type(func_response) == tuple and type(func_response[1]) == int and func_response[1] >= 100 and func_response[1] < 599:
|
|
159
|
-
status_code = func_response[1]
|
|
160
|
-
elif type(func_response) == int and func_response >= 100 and func_response < 599:
|
|
161
|
-
status_code = func_response
|
|
130
|
+
user = datastore_client.get_entity(USERS_KIND, int(db_user_id))
|
|
131
|
+
if user is None:
|
|
132
|
+
authorized_clients = []
|
|
133
|
+
else:
|
|
134
|
+
authorized_clients = user.get('authorized_clients', [])
|
|
135
|
+
|
|
136
|
+
if ALL_CLIENTS_RIGHTS in authorized_clients:
|
|
137
|
+
authorized_clients.remove(ALL_CLIENTS_RIGHTS)
|
|
138
|
+
clients_query = datastore_client.query()
|
|
139
|
+
clients_query.kind = CLIENTS_KIND
|
|
140
|
+
clients_query.keys_only()
|
|
141
|
+
authorized_clients = [client_entity.key.name for client_entity in clients_query.fetch()]
|
|
142
|
+
|
|
143
|
+
kwargs[CLIENTS_PARAM] = sorted(set(authorized_clients))
|
|
162
144
|
|
|
163
145
|
try:
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
146
|
+
if asyncio.iscoroutinefunction(job_func):
|
|
147
|
+
func_response = await asyncio.wait_for(job_func(*args, **kwargs), timeout=timeout)
|
|
148
|
+
else:
|
|
149
|
+
# Synchronous function - wrap in executor for timeout handling
|
|
150
|
+
if timeout is not None:
|
|
151
|
+
loop = asyncio.get_event_loop()
|
|
152
|
+
func_response = await asyncio.wait_for(
|
|
153
|
+
loop.run_in_executor(None, lambda: job_func(*args, **kwargs)),
|
|
154
|
+
timeout=timeout
|
|
155
|
+
)
|
|
156
|
+
else:
|
|
157
|
+
func_response = job_func(*args, **kwargs)
|
|
158
|
+
inner_ctx = tracking_ctx.get()
|
|
159
|
+
if inner_ctx is not None:
|
|
160
|
+
_inner_additional_info[0] = inner_ctx.additional_info
|
|
161
|
+
return func_response
|
|
162
|
+
except asyncio.TimeoutError:
|
|
163
|
+
return {'detail': 'Request Timeout'}, 504
|
|
164
|
+
except Exception as e:
|
|
165
|
+
return {'detail': f'Server Error: {str(e)}'}, 500
|
|
166
|
+
|
|
167
|
+
# Run the async execution in a new event loop
|
|
168
|
+
func_response = asyncio.run(async_execution())
|
|
169
|
+
|
|
170
|
+
additionnal_info = {key: get(kwargs, path) for key, path in kwargs_name_and_path_to_get.items()}
|
|
171
|
+
# Merge enrich_tracking() data from inside the route: asyncio.run() isolates ContextVar writes
|
|
172
|
+
# to a copy of the context, so we capture it via closure and merge here.
|
|
173
|
+
# Kwargs-derived paths take precedence over enrich_tracking() data.
|
|
174
|
+
if _inner_additional_info[0]:
|
|
175
|
+
additionnal_info = {**_inner_additional_info[0], **additionnal_info}
|
|
176
|
+
# Set tracking context here (same context as after_request) so init_tracking can read it
|
|
177
|
+
if claims or origin:
|
|
178
|
+
ctx = tracking_ctx.get()
|
|
179
|
+
if ctx is None:
|
|
180
|
+
ctx = TrackingContext()
|
|
181
|
+
tracking_ctx.set(ctx)
|
|
182
|
+
ctx.email = claims.get('email', '')
|
|
183
|
+
ctx.origin = origin
|
|
184
|
+
ctx.service = service
|
|
185
|
+
ctx.function_name = job_func.__name__
|
|
186
|
+
ctx = tracking_ctx.get()
|
|
187
|
+
if ctx is not None:
|
|
188
|
+
ctx.additional_info = additionnal_info or None
|
|
189
|
+
g.tracking_additional_info = additionnal_info or None # backward compat
|
|
178
190
|
|
|
179
191
|
return func_response
|
|
180
192
|
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Tracking package: ports & adapters for activity tracking."""
|
|
2
|
+
|
|
3
|
+
from .adapters import LogTrackingAdapter, PubSubTrackingAdapter, DEFAULT_COLUMNS
|
|
4
|
+
from .context import TrackingContext, tracking_ctx, enrich_tracking
|
|
5
|
+
from .events import ActivityEvent
|
|
6
|
+
from .flask_integration import init_tracking
|
|
7
|
+
from .ports import TrackingPort
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"ActivityEvent",
|
|
11
|
+
"TrackingPort",
|
|
12
|
+
"TrackingContext",
|
|
13
|
+
"tracking_ctx",
|
|
14
|
+
"enrich_tracking",
|
|
15
|
+
"init_tracking",
|
|
16
|
+
"LogTrackingAdapter",
|
|
17
|
+
"PubSubTrackingAdapter",
|
|
18
|
+
"DEFAULT_COLUMNS"
|
|
19
|
+
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Log adapter: emits tracking events as structured log lines at INFO."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
|
|
6
|
+
from ..ports import TrackingPort
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from ..events import ActivityEvent
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class LogTrackingAdapter(TrackingPort):
|
|
15
|
+
"""Emits each tracking event as a single INFO log line with key fields.
|
|
16
|
+
This is used for testing / mocking
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def emit(self, event: "ActivityEvent") -> None:
|
|
20
|
+
extra = ""
|
|
21
|
+
if event.additional_info:
|
|
22
|
+
extra = " " + str(event.additional_info)
|
|
23
|
+
logger.info(
|
|
24
|
+
"TRACK %s %s/%s [%s] %.3fs%s",
|
|
25
|
+
event.email,
|
|
26
|
+
event.service,
|
|
27
|
+
event.function_name,
|
|
28
|
+
event.status_code,
|
|
29
|
+
event.execution_time,
|
|
30
|
+
extra,
|
|
31
|
+
)
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""PubSub adapter: emits tracking events to Google Cloud Pub/Sub."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Optional, List, Any
|
|
5
|
+
|
|
6
|
+
from ..events import ActivityEvent
|
|
7
|
+
from ..ports import TrackingPort
|
|
8
|
+
import logging
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
# Wire-format keys (additionnal_info matches existing consumer schema)
|
|
13
|
+
DEFAULT_COLUMNS = (
|
|
14
|
+
"user_email",
|
|
15
|
+
"service",
|
|
16
|
+
"execution_time",
|
|
17
|
+
"additionnal_info",
|
|
18
|
+
"status_code",
|
|
19
|
+
"timestamp",
|
|
20
|
+
"function_name",
|
|
21
|
+
"insert_id",
|
|
22
|
+
"origin",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _event_to_parameters(event: ActivityEvent) -> dict:
|
|
27
|
+
"""Build the full parameters dict from an ActivityEvent (wire format)."""
|
|
28
|
+
return {
|
|
29
|
+
"user_email": event.email,
|
|
30
|
+
"service": event.service,
|
|
31
|
+
"execution_time": event.execution_time,
|
|
32
|
+
"additionnal_info": json.dumps(event.additional_info) if event.additional_info is not None else None,
|
|
33
|
+
"status_code": event.status_code,
|
|
34
|
+
"timestamp": event.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
35
|
+
"function_name": event.function_name,
|
|
36
|
+
"insert_id": event.insert_id,
|
|
37
|
+
"origin": event.origin,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class PubSubTrackingAdapter(TrackingPort):
|
|
42
|
+
"""Emits tracking events to a Pub/Sub topic. Optional columns filter which fields are sent."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
pubsub_client: Any,
|
|
47
|
+
project: str,
|
|
48
|
+
topic: str = "activity-tracker",
|
|
49
|
+
columns: Optional[List[str]] = None,
|
|
50
|
+
):
|
|
51
|
+
self._client = pubsub_client
|
|
52
|
+
self._project = project
|
|
53
|
+
self._topic = topic
|
|
54
|
+
self._columns = list(columns) if columns is not None else list(DEFAULT_COLUMNS)
|
|
55
|
+
|
|
56
|
+
def emit(self, event: ActivityEvent) -> None:
|
|
57
|
+
params = _event_to_parameters(event)
|
|
58
|
+
logger.debug(f"PubSubTrackingAdapter: emitting event: {params}")
|
|
59
|
+
filtered = {k: params[k] for k in self._columns if k in params}
|
|
60
|
+
self._client.push_to_topic(
|
|
61
|
+
project=self._project,
|
|
62
|
+
topic_name=self._topic,
|
|
63
|
+
parameters=filtered,
|
|
64
|
+
)
|
|
65
|
+
logger.debug(f"PubSubTrackingAdapter: event emitted for topic {self._topic}: {filtered}")
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Request-scoped tracking context (framework-agnostic, uses contextvars)."""
|
|
2
|
+
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Optional, Dict, Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class TrackingContext:
|
|
10
|
+
"""
|
|
11
|
+
Mutable accumulator for tracking data during a request.
|
|
12
|
+
Populated by auth decorator and/or enrich_tracking(); read by init_tracking in after_request.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
email: str = ""
|
|
16
|
+
service: str = ""
|
|
17
|
+
function_name: str = ""
|
|
18
|
+
origin: str = ""
|
|
19
|
+
additional_info: Optional[Dict[str, Any]] = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
tracking_ctx: ContextVar[Optional[TrackingContext]] = ContextVar(
|
|
23
|
+
"tracking_ctx",
|
|
24
|
+
default=None,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def enrich_tracking(additional_info: dict[str, Any]) -> None:
|
|
29
|
+
"""
|
|
30
|
+
Merge key/value pairs into the current request's tracking context to enrich additional_info field
|
|
31
|
+
Creates the context lazily if it does not exist (safe for undecorated routes).
|
|
32
|
+
Multiple calls accumulate; later values overwrite same key.
|
|
33
|
+
"""
|
|
34
|
+
ctx = tracking_ctx.get()
|
|
35
|
+
if ctx is None:
|
|
36
|
+
ctx = TrackingContext()
|
|
37
|
+
tracking_ctx.set(ctx)
|
|
38
|
+
if ctx.additional_info is None:
|
|
39
|
+
ctx.additional_info = {}
|
|
40
|
+
ctx.additional_info.update(additional_info)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Centralized schema for tracking events (framework-agnostic)."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Optional, Dict
|
|
6
|
+
import uuid
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class ActivityEvent:
|
|
11
|
+
"""
|
|
12
|
+
Immutable snapshot of one tracking event.
|
|
13
|
+
All adapters serialize from this; all producers create it.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
email: str
|
|
17
|
+
service: str
|
|
18
|
+
function_name: str
|
|
19
|
+
origin: str
|
|
20
|
+
status_code: int
|
|
21
|
+
execution_time: float
|
|
22
|
+
additional_info: Optional[dict] = None
|
|
23
|
+
timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
24
|
+
insert_id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Flask-specific wiring for tracking: request hooks that read context and emit via TrackingPort."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import time
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from flask import g, request
|
|
8
|
+
|
|
9
|
+
from .context import tracking_ctx, TrackingContext
|
|
10
|
+
from .events import ActivityEvent
|
|
11
|
+
from .ports import TrackingPort
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from flask import Flask
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
# Endpoints that should not be tracked (e.g. health checks)
|
|
19
|
+
EXCLUDED_ENDPOINTS = frozenset()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def init_tracking(app: "Flask", tracker: TrackingPort, service_name: str, excluded_endpoints: frozenset = None) -> None:
|
|
23
|
+
"""
|
|
24
|
+
Register before_request and after_request hooks to track every request.
|
|
25
|
+
Uses TrackingContext (populated by auth decorator and/or enrich_tracking).
|
|
26
|
+
If no context was set, builds a baseline from the request (function_name from endpoint, etc.).
|
|
27
|
+
"""
|
|
28
|
+
excluded = excluded_endpoints if excluded_endpoints is not None else EXCLUDED_ENDPOINTS
|
|
29
|
+
|
|
30
|
+
@app.before_request
|
|
31
|
+
def _start_timer():
|
|
32
|
+
tracking_ctx.set(None)
|
|
33
|
+
g.request_start = time.time()
|
|
34
|
+
logger.debug("Tracking: request started, timer set.")
|
|
35
|
+
|
|
36
|
+
@app.after_request
|
|
37
|
+
def _track_request(response):
|
|
38
|
+
if request.endpoint in excluded:
|
|
39
|
+
return response
|
|
40
|
+
|
|
41
|
+
ctx = tracking_ctx.get()
|
|
42
|
+
if ctx is None:
|
|
43
|
+
ctx = TrackingContext(
|
|
44
|
+
function_name=request.endpoint or request.path,
|
|
45
|
+
origin=request.headers.get("Origin", ""),
|
|
46
|
+
service=service_name,
|
|
47
|
+
email="",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
execution_time = time.time() - getattr(g, "request_start", time.time())
|
|
51
|
+
event = ActivityEvent(
|
|
52
|
+
email=ctx.email,
|
|
53
|
+
service=ctx.service or service_name,
|
|
54
|
+
function_name=ctx.function_name or (request.endpoint or request.path),
|
|
55
|
+
origin=ctx.origin,
|
|
56
|
+
status_code=response.status_code,
|
|
57
|
+
execution_time=execution_time,
|
|
58
|
+
additional_info=ctx.additional_info,
|
|
59
|
+
)
|
|
60
|
+
try:
|
|
61
|
+
tracker.emit(event)
|
|
62
|
+
logger.debug(f"Tracking: event emitted: {event} for request {request.endpoint}")
|
|
63
|
+
except Exception:
|
|
64
|
+
logger.exception("Tracking: failed to emit event for request")
|
|
65
|
+
return response
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Port (abstraction) for emitting tracking events."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
|
|
5
|
+
from .events import ActivityEvent
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TrackingPort(ABC):
|
|
9
|
+
"""Abstract port for emitting activity tracking events. Implementations (adapters) handle transport."""
|
|
10
|
+
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def emit(self, event: ActivityEvent) -> None:
|
|
13
|
+
"""Emit a tracking event to the configured sink."""
|
|
14
|
+
...
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
from datetime import datetime
|
|
2
|
-
import pytz
|
|
3
|
-
import uuid
|
|
4
|
-
import json
|
|
5
|
-
from typing import Optional, Dict
|
|
6
|
-
|
|
7
|
-
from .log import adscale_log
|
|
8
|
-
from .services import ServiceEnum
|
|
9
|
-
from arcane.datastore import Client as DatastoreClient
|
|
10
|
-
from arcane.pubsub import Client as PubSubClient
|
|
11
|
-
|
|
12
|
-
PUBSUB_ACTIVITY_TRACKING_TOPIC = 'activity-tracker'
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def send_tracking_information(
|
|
16
|
-
email: str,
|
|
17
|
-
service: str,
|
|
18
|
-
function_name: str,
|
|
19
|
-
origin: str,
|
|
20
|
-
project: str,
|
|
21
|
-
execution_time: float,
|
|
22
|
-
additionnal_info: Optional[Dict],
|
|
23
|
-
status_code: int,
|
|
24
|
-
pubsub_client: PubSubClient,
|
|
25
|
-
datastore_client: DatastoreClient) -> None:
|
|
26
|
-
""" Send pubsub message to the tracking cloud function"""
|
|
27
|
-
|
|
28
|
-
users_query = datastore_client.query(kind='users')
|
|
29
|
-
users_query.add_filter('email', '=', email)
|
|
30
|
-
if users_query is None:
|
|
31
|
-
adscale_log(ServiceEnum.TRACKING, f'User {email} is not an Adscale User. Tracking info will not be saved.')
|
|
32
|
-
else:
|
|
33
|
-
timestamp = datetime.now(tz=pytz.timezone('Europe/Paris')).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
34
|
-
insert_id = str(uuid.uuid4())
|
|
35
|
-
|
|
36
|
-
pubsub_client.push_to_topic(
|
|
37
|
-
project=project,
|
|
38
|
-
topic_name=PUBSUB_ACTIVITY_TRACKING_TOPIC,
|
|
39
|
-
parameters={'user_email': email,
|
|
40
|
-
'service': service,
|
|
41
|
-
'execution_time': execution_time,
|
|
42
|
-
'additionnal_info': json.dumps(additionnal_info) if additionnal_info is not None else None,
|
|
43
|
-
'status_code': status_code,
|
|
44
|
-
'timestamp': timestamp,
|
|
45
|
-
'function_name': function_name,
|
|
46
|
-
'insert_id': insert_id,
|
|
47
|
-
'origin': origin
|
|
48
|
-
}
|
|
49
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|