tracelet-sdk 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tracelet_sdk-0.1.0/PKG-INFO +68 -0
- tracelet_sdk-0.1.0/README.md +58 -0
- tracelet_sdk-0.1.0/pyproject.toml +19 -0
- tracelet_sdk-0.1.0/setup.cfg +4 -0
- tracelet_sdk-0.1.0/tests/test_sdk.py +237 -0
- tracelet_sdk-0.1.0/tracelet_sdk/__init__.py +130 -0
- tracelet_sdk-0.1.0/tracelet_sdk/asgi.py +36 -0
- tracelet_sdk-0.1.0/tracelet_sdk/django.py +37 -0
- tracelet_sdk-0.1.0/tracelet_sdk/flask.py +21 -0
- tracelet_sdk-0.1.0/tracelet_sdk/sqlalchemy.py +20 -0
- tracelet_sdk-0.1.0/tracelet_sdk/wsgi.py +26 -0
- tracelet_sdk-0.1.0/tracelet_sdk.egg-info/PKG-INFO +68 -0
- tracelet_sdk-0.1.0/tracelet_sdk.egg-info/SOURCES.txt +13 -0
- tracelet_sdk-0.1.0/tracelet_sdk.egg-info/dependency_links.txt +1 -0
- tracelet_sdk-0.1.0/tracelet_sdk.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tracelet-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tag your database queries with the request that ran them, so Tracelet can show per-request waterfalls and catch N+1 loops.
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/moseleydev/tracelet
|
|
7
|
+
Keywords: tracelet,sqlcommenter,postgres,database,performance,n+1
|
|
8
|
+
Requires-Python: >=3.8
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# tracelet-sdk (Python)
|
|
12
|
+
|
|
13
|
+
Tags each database query with the request that ran it, so [Tracelet](https://github.com/moseleydev/tracelet) can show per-request waterfalls and catch N+1 loops. Zero dependencies. Python 3.8+.
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install tracelet-sdk
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Django
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
MIDDLEWARE = [
|
|
23
|
+
"tracelet_sdk.django.TraceletMiddleware",
|
|
24
|
+
# ...
|
|
25
|
+
]
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
That's all. Every query Django runs during a request carries the URL pattern (`GET /users/<int:pk>/`) and a trace id.
|
|
29
|
+
|
|
30
|
+
## Flask
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from tracelet_sdk.flask import instrument
|
|
34
|
+
from tracelet_sdk.sqlalchemy import instrument as instrument_engine
|
|
35
|
+
|
|
36
|
+
instrument(app)
|
|
37
|
+
instrument_engine(db.engine) # Flask-SQLAlchemy, or any SQLAlchemy engine
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## FastAPI / Starlette / any ASGI app
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from tracelet_sdk.asgi import TraceletMiddleware
|
|
44
|
+
from tracelet_sdk.sqlalchemy import instrument
|
|
45
|
+
|
|
46
|
+
app.add_middleware(TraceletMiddleware) # reads the matched FastAPI route
|
|
47
|
+
instrument(engine)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Plain WSGI apps can use `tracelet_sdk.wsgi.TraceletMiddleware(app, route=...)`.
|
|
51
|
+
|
|
52
|
+
## Anything else
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import tracelet_sdk
|
|
56
|
+
|
|
57
|
+
with tracelet_sdk.request(route="POST /import"):
|
|
58
|
+
cursor.execute(tracelet_sdk.comment("SELECT * FROM jobs WHERE id = %s", escape_percent=True), (job_id,))
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Pass `escape_percent=True` when the driver %-formats the query (psycopg, MySQLdb and Django with parameters). The comment's URL-encoded values contain `%`, which must then be doubled. The Django and SQLAlchemy integrations do this for you.
|
|
62
|
+
|
|
63
|
+
## Notes
|
|
64
|
+
|
|
65
|
+
- **Traces:** an incoming `traceparent` header is continued, so traces line up with OpenTelemetry.
|
|
66
|
+
- **No raw URLs:** paths are never recorded, so ids and other values in them don't end up in your stats.
|
|
67
|
+
- **Existing comments:** SQL that already contains a comment is left unchanged, as the sqlcommenter spec requires.
|
|
68
|
+
- **Streaming responses:** in WSGI, queries made while a streamed response is iterated are not tagged.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# tracelet-sdk (Python)
|
|
2
|
+
|
|
3
|
+
Tags each database query with the request that ran it, so [Tracelet](https://github.com/moseleydev/tracelet) can show per-request waterfalls and catch N+1 loops. Zero dependencies. Python 3.8+.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install tracelet-sdk
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Django
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
MIDDLEWARE = [
|
|
13
|
+
"tracelet_sdk.django.TraceletMiddleware",
|
|
14
|
+
# ...
|
|
15
|
+
]
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
That's all. Every query Django runs during a request carries the URL pattern (`GET /users/<int:pk>/`) and a trace id.
|
|
19
|
+
|
|
20
|
+
## Flask
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from tracelet_sdk.flask import instrument
|
|
24
|
+
from tracelet_sdk.sqlalchemy import instrument as instrument_engine
|
|
25
|
+
|
|
26
|
+
instrument(app)
|
|
27
|
+
instrument_engine(db.engine) # Flask-SQLAlchemy, or any SQLAlchemy engine
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## FastAPI / Starlette / any ASGI app
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from tracelet_sdk.asgi import TraceletMiddleware
|
|
34
|
+
from tracelet_sdk.sqlalchemy import instrument
|
|
35
|
+
|
|
36
|
+
app.add_middleware(TraceletMiddleware) # reads the matched FastAPI route
|
|
37
|
+
instrument(engine)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Plain WSGI apps can use `tracelet_sdk.wsgi.TraceletMiddleware(app, route=...)`.
|
|
41
|
+
|
|
42
|
+
## Anything else
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
import tracelet_sdk
|
|
46
|
+
|
|
47
|
+
with tracelet_sdk.request(route="POST /import"):
|
|
48
|
+
cursor.execute(tracelet_sdk.comment("SELECT * FROM jobs WHERE id = %s", escape_percent=True), (job_id,))
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Pass `escape_percent=True` when the driver %-formats the query (psycopg, MySQLdb and Django with parameters). The comment's URL-encoded values contain `%`, which must then be doubled. The Django and SQLAlchemy integrations do this for you.
|
|
52
|
+
|
|
53
|
+
## Notes
|
|
54
|
+
|
|
55
|
+
- **Traces:** an incoming `traceparent` header is continued, so traces line up with OpenTelemetry.
|
|
56
|
+
- **No raw URLs:** paths are never recorded, so ids and other values in them don't end up in your stats.
|
|
57
|
+
- **Existing comments:** SQL that already contains a comment is left unchanged, as the sqlcommenter spec requires.
|
|
58
|
+
- **Streaming responses:** in WSGI, queries made while a streamed response is iterated are not tagged.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "tracelet-sdk"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Tag your database queries with the request that ran them, so Tracelet can show per-request waterfalls and catch N+1 loops."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "Apache-2.0" }
|
|
11
|
+
requires-python = ">=3.8"
|
|
12
|
+
dependencies = []
|
|
13
|
+
keywords = ["tracelet", "sqlcommenter", "postgres", "database", "performance", "n+1"]
|
|
14
|
+
|
|
15
|
+
[project.urls]
|
|
16
|
+
Homepage = "https://github.com/moseleydev/tracelet"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools]
|
|
19
|
+
packages = ["tracelet_sdk"]
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
import threading
|
|
7
|
+
import unittest
|
|
8
|
+
from contextvars import copy_context
|
|
9
|
+
|
|
10
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
11
|
+
|
|
12
|
+
import tracelet_sdk as sdk # noqa: E402
|
|
13
|
+
from tracelet_sdk import asgi, wsgi # noqa: E402
|
|
14
|
+
|
|
15
|
+
VECTORS = os.path.join(os.path.dirname(__file__), "..", "..", "testdata", "sqlcommenter.json")
|
|
16
|
+
TP = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def installed(module):
|
|
20
|
+
try:
|
|
21
|
+
__import__(module)
|
|
22
|
+
return True
|
|
23
|
+
except ImportError:
|
|
24
|
+
return False
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class CommentTest(unittest.TestCase):
|
|
28
|
+
def test_shared_vectors_byte_for_byte(self):
|
|
29
|
+
with open(VECTORS, encoding="utf-8") as f:
|
|
30
|
+
cases = json.load(f)["cases"]
|
|
31
|
+
for c in cases:
|
|
32
|
+
with self.subTest(c["name"]):
|
|
33
|
+
self.assertEqual(sdk.comment(c["sql"], c["fields"]), c["commented"])
|
|
34
|
+
|
|
35
|
+
def test_uses_current_request_and_is_noop_outside(self):
|
|
36
|
+
self.assertEqual(sdk.comment("SELECT 1"), "SELECT 1")
|
|
37
|
+
with sdk.request(route="GET /a", traceparent=TP):
|
|
38
|
+
self.assertEqual(sdk.comment("SELECT 1"), "SELECT 1 /*route='GET%20%2Fa',traceparent='{}'*/".format(TP))
|
|
39
|
+
self.assertIsNone(sdk.current_request())
|
|
40
|
+
|
|
41
|
+
def test_escape_percent_for_format_style_drivers(self):
|
|
42
|
+
out = sdk.comment("SELECT * FROM t WHERE id = %s", {"route": "/a b"}, escape_percent=True)
|
|
43
|
+
self.assertEqual(out, "SELECT * FROM t WHERE id = %s /*route='%%2Fa%%20b'*/")
|
|
44
|
+
self.assertEqual(out % ("1",), "SELECT * FROM t WHERE id = 1 /*route='%2Fa%20b'*/")
|
|
45
|
+
|
|
46
|
+
def test_lazy_route_errors_never_break_queries(self):
|
|
47
|
+
def broken():
|
|
48
|
+
raise RuntimeError("no route yet")
|
|
49
|
+
|
|
50
|
+
with sdk.request(route=broken, traceparent=TP):
|
|
51
|
+
self.assertEqual(sdk.comment("SELECT 1"), "SELECT 1 /*traceparent='{}'*/".format(TP))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class TraceparentTest(unittest.TestCase):
|
|
55
|
+
def test_parse(self):
|
|
56
|
+
self.assertEqual(sdk.parse_traceparent(TP)["trace_id"], "4bf92f3577b34da6a3ce929d0e0e4736")
|
|
57
|
+
for bad in (None, "", "nope", "00-" + "0" * 32 + "-00f067aa0ba902b7-01", "ff" + TP[2:]):
|
|
58
|
+
self.assertIsNone(sdk.parse_traceparent(bad), bad)
|
|
59
|
+
|
|
60
|
+
def test_new_continues_incoming_trace(self):
|
|
61
|
+
self.assertIsNotNone(sdk.parse_traceparent(sdk.new_traceparent()))
|
|
62
|
+
cont = sdk.parse_traceparent(sdk.new_traceparent(TP))
|
|
63
|
+
self.assertEqual(cont["trace_id"], "4bf92f3577b34da6a3ce929d0e0e4736")
|
|
64
|
+
self.assertNotEqual(cont["parent_id"], "00f067aa0ba902b7")
|
|
65
|
+
|
|
66
|
+
def test_invalid_traceparent_is_replaced(self):
|
|
67
|
+
with sdk.request(traceparent="garbage") as ctx:
|
|
68
|
+
self.assertIsNotNone(sdk.parse_traceparent(ctx.traceparent))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class ContextTest(unittest.TestCase):
|
|
72
|
+
def test_asyncio_tasks_keep_separate_context(self):
|
|
73
|
+
async def handle(route):
|
|
74
|
+
with sdk.request(route=route):
|
|
75
|
+
await asyncio.sleep(0.01)
|
|
76
|
+
return route, sdk.comment("SELECT 1")
|
|
77
|
+
|
|
78
|
+
async def main():
|
|
79
|
+
return await asyncio.gather(*(handle(r) for r in ("/a", "/b", "/c")))
|
|
80
|
+
|
|
81
|
+
for route, sql in asyncio.run(main()):
|
|
82
|
+
self.assertIn("route='%2F{}'".format(route[1:]), sql)
|
|
83
|
+
|
|
84
|
+
def test_threads_that_copy_context(self):
|
|
85
|
+
seen = []
|
|
86
|
+
with sdk.request(route="/t"):
|
|
87
|
+
ctx = copy_context()
|
|
88
|
+
t = threading.Thread(target=lambda: seen.append(ctx.run(sdk.comment, "SELECT 1")))
|
|
89
|
+
t.start()
|
|
90
|
+
t.join()
|
|
91
|
+
self.assertIn("route='%2Ft'", seen[0])
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class MiddlewareTest(unittest.TestCase):
|
|
95
|
+
def test_wsgi(self):
|
|
96
|
+
seen = {}
|
|
97
|
+
|
|
98
|
+
def app(environ, start_response):
|
|
99
|
+
seen["before"] = sdk.comment("SELECT 1")
|
|
100
|
+
environ["test.route"] = "GET /users/<id>" # what a router would set
|
|
101
|
+
seen["after"] = sdk.comment("SELECT 1")
|
|
102
|
+
start_response("200 OK", [])
|
|
103
|
+
return [b"ok"]
|
|
104
|
+
|
|
105
|
+
mw = wsgi.TraceletMiddleware(app, route=lambda env: env.get("test.route"))
|
|
106
|
+
body = mw({"HTTP_TRACEPARENT": TP, "PATH_INFO": "/users/42"}, lambda *a: None)
|
|
107
|
+
self.assertEqual(body, [b"ok"])
|
|
108
|
+
self.assertNotIn("route=", seen["before"])
|
|
109
|
+
self.assertIn("traceparent='00-4bf92f3577b34da6a3ce929d0e0e4736-", seen["after"])
|
|
110
|
+
self.assertIn("route='GET%20%2Fusers%2F%3Cid%3E'", seen["after"])
|
|
111
|
+
self.assertNotIn("%2F42", seen["after"]) # the raw path never leaks (ids may appear in the random span id)
|
|
112
|
+
|
|
113
|
+
def test_asgi_reads_fastapi_route_after_routing(self):
|
|
114
|
+
seen = {}
|
|
115
|
+
|
|
116
|
+
class Route:
|
|
117
|
+
path = "/items/{item_id}"
|
|
118
|
+
|
|
119
|
+
async def app(scope, receive, send):
|
|
120
|
+
seen["before"] = sdk.comment("SELECT 1")
|
|
121
|
+
scope["route"] = Route() # FastAPI sets this when a route matches
|
|
122
|
+
await asyncio.sleep(0)
|
|
123
|
+
seen["after"] = sdk.comment("SELECT 1")
|
|
124
|
+
|
|
125
|
+
scope = {"type": "http", "method": "GET", "path": "/items/7", "headers": [(b"traceparent", TP.encode())]}
|
|
126
|
+
asyncio.run(asgi.TraceletMiddleware(app)(scope, None, None))
|
|
127
|
+
self.assertNotIn("route=", seen["before"])
|
|
128
|
+
self.assertIn("route='GET%20%2Fitems%2F%7Bitem_id%7D'", seen["after"])
|
|
129
|
+
self.assertIn("4bf92f3577b34da6a3ce929d0e0e4736", seen["after"])
|
|
130
|
+
|
|
131
|
+
def test_asgi_passes_other_scopes_through(self):
|
|
132
|
+
calls = []
|
|
133
|
+
|
|
134
|
+
async def app(scope, receive, send):
|
|
135
|
+
calls.append(sdk.current_request())
|
|
136
|
+
|
|
137
|
+
asyncio.run(asgi.TraceletMiddleware(app)({"type": "lifespan"}, None, None))
|
|
138
|
+
self.assertEqual(calls, [None])
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@unittest.skipUnless(installed("sqlalchemy"), "sqlalchemy not installed")
|
|
142
|
+
class SQLAlchemyTest(unittest.TestCase):
|
|
143
|
+
def test_tags_statements_and_queries_still_run(self):
|
|
144
|
+
from sqlalchemy import create_engine, event, text
|
|
145
|
+
|
|
146
|
+
from tracelet_sdk.sqlalchemy import instrument
|
|
147
|
+
|
|
148
|
+
engine = instrument(create_engine("sqlite://"))
|
|
149
|
+
sent = []
|
|
150
|
+
event.listen(engine, "after_cursor_execute", lambda conn, cur, stmt, *a: sent.append(stmt))
|
|
151
|
+
with engine.connect() as conn:
|
|
152
|
+
with sdk.request(route="GET /r", traceparent=TP):
|
|
153
|
+
value = conn.execute(text("SELECT :x + 1"), {"x": 41}).scalar()
|
|
154
|
+
conn.execute(text("SELECT 1"))
|
|
155
|
+
self.assertEqual(value, 42)
|
|
156
|
+
self.assertRegex(sent[-2], r"^SELECT \? \+ 1 /\*route='GET%20%2Fr',traceparent='00-")
|
|
157
|
+
self.assertEqual(sent[-1], "SELECT 1")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@unittest.skipUnless(installed("django"), "django not installed")
|
|
161
|
+
class DjangoTest(unittest.TestCase):
|
|
162
|
+
@classmethod
|
|
163
|
+
def setUpClass(cls):
|
|
164
|
+
import django
|
|
165
|
+
from django.conf import settings
|
|
166
|
+
|
|
167
|
+
if not settings.configured:
|
|
168
|
+
settings.configure(
|
|
169
|
+
DATABASES={"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}},
|
|
170
|
+
ROOT_URLCONF=__name__,
|
|
171
|
+
ALLOWED_HOSTS=["*"],
|
|
172
|
+
)
|
|
173
|
+
django.setup()
|
|
174
|
+
|
|
175
|
+
def test_tags_queries_with_url_pattern(self):
|
|
176
|
+
from django.db import connection
|
|
177
|
+
from django.test import RequestFactory
|
|
178
|
+
from django.urls import resolve
|
|
179
|
+
|
|
180
|
+
from tracelet_sdk.django import TraceletMiddleware
|
|
181
|
+
|
|
182
|
+
sent = []
|
|
183
|
+
|
|
184
|
+
def get_response(request):
|
|
185
|
+
request.resolver_match = resolve("/users/42/")
|
|
186
|
+
|
|
187
|
+
def capture(execute, sql, params, many, context):
|
|
188
|
+
sent.append(sql)
|
|
189
|
+
return execute(sql, params, many, context)
|
|
190
|
+
|
|
191
|
+
with connection.execute_wrapper(capture), connection.cursor() as cur:
|
|
192
|
+
cur.execute("SELECT %s + 1", [41]) # %-formatted by the backend
|
|
193
|
+
value = cur.fetchone()[0]
|
|
194
|
+
cur.execute("SELECT 1") # no params: sent as-is
|
|
195
|
+
return value
|
|
196
|
+
|
|
197
|
+
req = RequestFactory().get("/users/42/", HTTP_TRACEPARENT=TP)
|
|
198
|
+
self.assertEqual(TraceletMiddleware(get_response)(req), 42)
|
|
199
|
+
self.assertRegex(sent[0], r"^SELECT %s \+ 1 /\*route='GET%%20%%2Fusers%%2F%%3Cint%%3Apk%%3E%%2F',traceparent='00-4bf92f")
|
|
200
|
+
self.assertRegex(sent[1], r"^SELECT 1 /\*route='GET%20%2Fusers")
|
|
201
|
+
self.assertNotIn("%%2F42", sent[0]) # the raw path never leaks
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@unittest.skipUnless(installed("flask"), "flask not installed")
|
|
205
|
+
class FlaskTest(unittest.TestCase):
|
|
206
|
+
def test_tags_with_flask_rule(self):
|
|
207
|
+
from flask import Flask
|
|
208
|
+
|
|
209
|
+
from tracelet_sdk.flask import instrument
|
|
210
|
+
|
|
211
|
+
app = instrument(Flask(__name__))
|
|
212
|
+
|
|
213
|
+
@app.get("/orders/<int:order_id>")
|
|
214
|
+
def order(order_id):
|
|
215
|
+
return sdk.comment("SELECT 1")
|
|
216
|
+
|
|
217
|
+
sql = app.test_client().get("/orders/9", headers={"traceparent": TP}).get_data(as_text=True)
|
|
218
|
+
self.assertRegex(sql, r"route='GET%20%2Forders%2F%3Cint%3Aorder_id%3E'")
|
|
219
|
+
self.assertIn("4bf92f3577b34da6a3ce929d0e0e4736", sql)
|
|
220
|
+
self.assertNotRegex(sql, re.escape("/orders/9"))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
# Django URLconf for DjangoTest.
|
|
224
|
+
def _view(request, pk):
|
|
225
|
+
return None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
try:
|
|
229
|
+
from django.urls import path
|
|
230
|
+
|
|
231
|
+
urlpatterns = [path("users/<int:pk>/", _view)]
|
|
232
|
+
except ImportError: # pragma: no cover
|
|
233
|
+
urlpatterns = []
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
if __name__ == "__main__":
|
|
237
|
+
unittest.main()
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Tag each database query with the request that ran it.
|
|
2
|
+
|
|
3
|
+
Queries get a sqlcommenter comment (https://google.github.io/sqlcommenter/)::
|
|
4
|
+
|
|
5
|
+
SELECT * FROM users WHERE id = %s /*route='GET%20%2Fusers%2F%3Aid',traceparent='00-…-…-01'*/
|
|
6
|
+
|
|
7
|
+
Tracelet reads these comments to group queries into requests. Nothing is
|
|
8
|
+
sent anywhere: the comment travels with the query to your database.
|
|
9
|
+
|
|
10
|
+
Framework integrations live in submodules: ``tracelet_sdk.wsgi``,
|
|
11
|
+
``tracelet_sdk.asgi``, ``tracelet_sdk.flask``, ``tracelet_sdk.django`` and
|
|
12
|
+
``tracelet_sdk.sqlalchemy``.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import contextvars
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
from contextlib import contextmanager
|
|
19
|
+
from typing import Callable, Dict, Iterator, Optional, Union
|
|
20
|
+
from urllib.parse import quote
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"RequestContext",
|
|
24
|
+
"request",
|
|
25
|
+
"current_request",
|
|
26
|
+
"comment",
|
|
27
|
+
"new_traceparent",
|
|
28
|
+
"parse_traceparent",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
Route = Union[str, Callable[[], Optional[str]], None]
|
|
32
|
+
|
|
33
|
+
_TRACEPARENT = re.compile(r"^([0-9a-f]{2})-([0-9a-f]{32})-([0-9a-f]{16})-([0-9a-f]{2})$")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class RequestContext:
|
|
37
|
+
"""The request a query belongs to."""
|
|
38
|
+
|
|
39
|
+
__slots__ = ("_route", "traceparent")
|
|
40
|
+
|
|
41
|
+
def __init__(self, route: Route = None, traceparent: Optional[str] = None):
|
|
42
|
+
self._route = route
|
|
43
|
+
self.traceparent = traceparent if parse_traceparent(traceparent) else new_traceparent(traceparent)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def route(self) -> Optional[str]:
|
|
47
|
+
"""The route pattern. Callables are evaluated now, so frameworks that
|
|
48
|
+
only know the route after routing can supply it lazily."""
|
|
49
|
+
route = self._route
|
|
50
|
+
if callable(route):
|
|
51
|
+
try:
|
|
52
|
+
route = route()
|
|
53
|
+
except Exception: # never let tagging break a query
|
|
54
|
+
route = None
|
|
55
|
+
return route or None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_current: "contextvars.ContextVar[Optional[RequestContext]]" = contextvars.ContextVar("tracelet_request", default=None)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def parse_traceparent(header: Optional[str]) -> Optional[Dict[str, str]]:
|
|
62
|
+
"""Parse a W3C traceparent header, or return None if it is not valid."""
|
|
63
|
+
if not isinstance(header, str):
|
|
64
|
+
return None
|
|
65
|
+
m = _TRACEPARENT.match(header.strip())
|
|
66
|
+
if not m or m.group(1) == "ff" or set(m.group(2)) == {"0"} or set(m.group(3)) == {"0"}:
|
|
67
|
+
return None
|
|
68
|
+
return {"version": m.group(1), "trace_id": m.group(2), "parent_id": m.group(3), "flags": m.group(4)}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def new_traceparent(incoming: Optional[str] = None) -> str:
|
|
72
|
+
"""A traceparent for a new request span. An incoming header's trace id is
|
|
73
|
+
kept, so Tracelet's traces line up with the rest of your tracing."""
|
|
74
|
+
parent = parse_traceparent(incoming)
|
|
75
|
+
trace_id = parent["trace_id"] if parent else os.urandom(16).hex()
|
|
76
|
+
flags = parent["flags"] if parent else "01"
|
|
77
|
+
return "00-{}-{}-{}".format(trace_id, os.urandom(8).hex(), flags)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@contextmanager
|
|
81
|
+
def request(route: Route = None, traceparent: Optional[str] = None) -> Iterator[RequestContext]:
|
|
82
|
+
"""Run a block as one request: queries inside it are tagged with it.
|
|
83
|
+
Works across ``await`` and in threads that copy the context."""
|
|
84
|
+
ctx = RequestContext(route, traceparent)
|
|
85
|
+
token = _current.set(ctx)
|
|
86
|
+
try:
|
|
87
|
+
yield ctx
|
|
88
|
+
finally:
|
|
89
|
+
_current.reset(token)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def current_request() -> Optional[RequestContext]:
|
|
93
|
+
"""The current request context, or None outside a request."""
|
|
94
|
+
return _current.get()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _enc(value: str) -> str:
|
|
98
|
+
# Same as JavaScript's encodeURIComponent, then sqlcommenter's \' escape.
|
|
99
|
+
return quote(str(value), safe="-_.!~*'()").replace("'", "\\'")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def comment(sql: str, fields: Optional[Dict[str, Optional[str]]] = None, *, escape_percent: bool = False) -> str:
|
|
103
|
+
"""Append a sqlcommenter comment with ``fields`` (or the current
|
|
104
|
+
request's) to ``sql``.
|
|
105
|
+
|
|
106
|
+
Set ``escape_percent`` when the driver will %-format the SQL (psycopg,
|
|
107
|
+
MySQLdb and Django with parameters): the comment's URL-encoded values
|
|
108
|
+
contain ``%``, which must then be written as ``%%``.
|
|
109
|
+
|
|
110
|
+
SQL that already contains a comment is returned unchanged, as the
|
|
111
|
+
sqlcommenter spec requires.
|
|
112
|
+
"""
|
|
113
|
+
if not isinstance(sql, str) or "*/" in sql:
|
|
114
|
+
return sql
|
|
115
|
+
if fields is None:
|
|
116
|
+
ctx = _current.get()
|
|
117
|
+
if ctx is None:
|
|
118
|
+
return sql
|
|
119
|
+
fields = {"route": ctx.route, "traceparent": ctx.traceparent}
|
|
120
|
+
pairs = ["{}='{}'".format(_enc(k), _enc(v)) for k, v in sorted(fields.items()) if v not in (None, "")]
|
|
121
|
+
if not pairs:
|
|
122
|
+
return sql
|
|
123
|
+
|
|
124
|
+
text = "/*" + ",".join(pairs) + "*/"
|
|
125
|
+
if escape_percent:
|
|
126
|
+
text = text.replace("%", "%%")
|
|
127
|
+
trimmed = sql.rstrip()
|
|
128
|
+
if trimmed.endswith(";"):
|
|
129
|
+
return "{} {};".format(trimmed[:-1].rstrip(), text)
|
|
130
|
+
return "{} {}".format(trimmed, text)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""ASGI middleware: one request context per HTTP request (FastAPI, Starlette, Django ASGI)."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable, Dict, Optional
|
|
4
|
+
|
|
5
|
+
from . import new_traceparent, request
|
|
6
|
+
|
|
7
|
+
RouteGetter = Callable[[Dict[str, Any]], Optional[str]]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def default_route(scope: Dict[str, Any]) -> Optional[str]:
|
|
11
|
+
"""The route pattern FastAPI records in the scope after routing."""
|
|
12
|
+
route = scope.get("route")
|
|
13
|
+
path = getattr(route, "path", None) or getattr(route, "path_format", None)
|
|
14
|
+
return "{} {}".format(scope.get("method", ""), path).strip() if path else None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TraceletMiddleware:
|
|
18
|
+
"""Wrap any ASGI app. ``route`` is called with the ASGI scope each time a
|
|
19
|
+
query runs; the default reads the route FastAPI matched."""
|
|
20
|
+
|
|
21
|
+
def __init__(self, app: Callable, route: Optional[RouteGetter] = default_route):
|
|
22
|
+
self.app = app
|
|
23
|
+
self.route = route
|
|
24
|
+
|
|
25
|
+
async def __call__(self, scope: Dict[str, Any], receive: Callable, send: Callable) -> None:
|
|
26
|
+
if scope.get("type") != "http":
|
|
27
|
+
await self.app(scope, receive, send)
|
|
28
|
+
return
|
|
29
|
+
incoming = None
|
|
30
|
+
for name, value in scope.get("headers") or []:
|
|
31
|
+
if name == b"traceparent":
|
|
32
|
+
incoming = value.decode("latin-1")
|
|
33
|
+
break
|
|
34
|
+
getter = (lambda: self.route(scope)) if self.route else None
|
|
35
|
+
with request(route=getter, traceparent=new_traceparent(incoming)):
|
|
36
|
+
await self.app(scope, receive, send)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Django integration. Add to MIDDLEWARE::
|
|
2
|
+
|
|
3
|
+
MIDDLEWARE = ["tracelet_sdk.django.TraceletMiddleware", ...]
|
|
4
|
+
|
|
5
|
+
Every query Django runs during a request is tagged with the URL pattern
|
|
6
|
+
("GET /users/<int:pk>/"). No other setup is needed.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from contextlib import ExitStack
|
|
10
|
+
from typing import Any, Callable
|
|
11
|
+
|
|
12
|
+
from . import comment, new_traceparent, request
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _execute_wrapper(execute: Callable, sql: str, params: Any, many: bool, context: Any) -> Any:
|
|
16
|
+
# With parameters, Django's backends %-format the SQL, so the comment's
|
|
17
|
+
# % signs must be doubled. Without parameters they are sent as-is.
|
|
18
|
+
return execute(comment(sql, escape_percent=params is not None), params, many, context)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TraceletMiddleware:
|
|
22
|
+
def __init__(self, get_response: Callable):
|
|
23
|
+
self.get_response = get_response
|
|
24
|
+
|
|
25
|
+
def __call__(self, http_request: Any) -> Any:
|
|
26
|
+
from django.db import connections
|
|
27
|
+
|
|
28
|
+
def route():
|
|
29
|
+
match = getattr(http_request, "resolver_match", None)
|
|
30
|
+
pattern = getattr(match, "route", None)
|
|
31
|
+
return "{} /{}".format(http_request.method, pattern) if pattern is not None else None
|
|
32
|
+
|
|
33
|
+
incoming = http_request.META.get("HTTP_TRACEPARENT")
|
|
34
|
+
with request(route=route, traceparent=new_traceparent(incoming)), ExitStack() as stack:
|
|
35
|
+
for conn in connections.all():
|
|
36
|
+
stack.enter_context(conn.execute_wrapper(_execute_wrapper))
|
|
37
|
+
return self.get_response(http_request)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Flask integration: ``tracelet_sdk.flask.instrument(app)``."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Dict, Optional
|
|
4
|
+
|
|
5
|
+
from .wsgi import TraceletMiddleware
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _route(environ: Dict[str, Any]) -> Optional[str]:
|
|
9
|
+
from flask import has_request_context, request
|
|
10
|
+
|
|
11
|
+
if has_request_context() and request.url_rule is not None:
|
|
12
|
+
return "{} {}".format(request.method, request.url_rule.rule)
|
|
13
|
+
return None
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def instrument(app: Any) -> Any:
|
|
17
|
+
"""Tag queries with the Flask route that ran them ("GET /users/<int:id>").
|
|
18
|
+
Combine with ``tracelet_sdk.sqlalchemy.instrument`` (Flask-SQLAlchemy:
|
|
19
|
+
``instrument(db.engine)``) or call ``tracelet_sdk.comment`` yourself."""
|
|
20
|
+
app.wsgi_app = TraceletMiddleware(app.wsgi_app, route=_route)
|
|
21
|
+
return app
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""SQLAlchemy integration: ``tracelet_sdk.sqlalchemy.instrument(engine)``."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from . import comment
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def instrument(engine: Any) -> Any:
|
|
9
|
+
"""Tag every statement the engine runs inside a request context.
|
|
10
|
+
Works with any driver; % signs are escaped for drivers that %-format."""
|
|
11
|
+
from sqlalchemy import event
|
|
12
|
+
|
|
13
|
+
target = getattr(engine, "sync_engine", engine) # AsyncEngine wraps a sync engine
|
|
14
|
+
|
|
15
|
+
@event.listens_for(target, "before_cursor_execute", retval=True)
|
|
16
|
+
def _tag(conn, cursor, statement, parameters, context, executemany):
|
|
17
|
+
formats = conn.dialect.paramstyle in ("format", "pyformat") and parameters is not None
|
|
18
|
+
return comment(statement, escape_percent=formats), parameters
|
|
19
|
+
|
|
20
|
+
return engine
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""WSGI middleware: one request context per HTTP request."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Callable, Dict, Optional
|
|
4
|
+
|
|
5
|
+
from . import new_traceparent, request
|
|
6
|
+
|
|
7
|
+
RouteGetter = Callable[[Dict[str, Any]], Optional[str]]
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TraceletMiddleware:
|
|
11
|
+
"""Wrap any WSGI app. ``route`` is called with the WSGI environ each
|
|
12
|
+
time a query runs and should return the matched route pattern (for
|
|
13
|
+
example ``"GET /users/<id>"``). Without it, queries are still grouped by
|
|
14
|
+
request, just without a route. Raw URLs are never used, so ids and other
|
|
15
|
+
values in paths don't end up in your stats."""
|
|
16
|
+
|
|
17
|
+
def __init__(self, app: Callable, route: Optional[RouteGetter] = None):
|
|
18
|
+
self.app = app
|
|
19
|
+
self.route = route
|
|
20
|
+
|
|
21
|
+
def __call__(self, environ: Dict[str, Any], start_response: Callable):
|
|
22
|
+
getter = (lambda: self.route(environ)) if self.route else None
|
|
23
|
+
with request(route=getter, traceparent=new_traceparent(environ.get("HTTP_TRACEPARENT"))):
|
|
24
|
+
# Queries made while a streamed response is being iterated, after
|
|
25
|
+
# this returns, are not tagged.
|
|
26
|
+
return self.app(environ, start_response)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tracelet-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Tag your database queries with the request that ran them, so Tracelet can show per-request waterfalls and catch N+1 loops.
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
Project-URL: Homepage, https://github.com/moseleydev/tracelet
|
|
7
|
+
Keywords: tracelet,sqlcommenter,postgres,database,performance,n+1
|
|
8
|
+
Requires-Python: >=3.8
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# tracelet-sdk (Python)
|
|
12
|
+
|
|
13
|
+
Tags each database query with the request that ran it, so [Tracelet](https://github.com/moseleydev/tracelet) can show per-request waterfalls and catch N+1 loops. Zero dependencies. Python 3.8+.
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install tracelet-sdk
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Django
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
MIDDLEWARE = [
|
|
23
|
+
"tracelet_sdk.django.TraceletMiddleware",
|
|
24
|
+
# ...
|
|
25
|
+
]
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
That's all. Every query Django runs during a request carries the URL pattern (`GET /users/<int:pk>/`) and a trace id.
|
|
29
|
+
|
|
30
|
+
## Flask
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from tracelet_sdk.flask import instrument
|
|
34
|
+
from tracelet_sdk.sqlalchemy import instrument as instrument_engine
|
|
35
|
+
|
|
36
|
+
instrument(app)
|
|
37
|
+
instrument_engine(db.engine) # Flask-SQLAlchemy, or any SQLAlchemy engine
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## FastAPI / Starlette / any ASGI app
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from tracelet_sdk.asgi import TraceletMiddleware
|
|
44
|
+
from tracelet_sdk.sqlalchemy import instrument
|
|
45
|
+
|
|
46
|
+
app.add_middleware(TraceletMiddleware) # reads the matched FastAPI route
|
|
47
|
+
instrument(engine)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Plain WSGI apps can use `tracelet_sdk.wsgi.TraceletMiddleware(app, route=...)`.
|
|
51
|
+
|
|
52
|
+
## Anything else
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import tracelet_sdk
|
|
56
|
+
|
|
57
|
+
with tracelet_sdk.request(route="POST /import"):
|
|
58
|
+
cursor.execute(tracelet_sdk.comment("SELECT * FROM jobs WHERE id = %s", escape_percent=True), (job_id,))
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Pass `escape_percent=True` when the driver %-formats the query (psycopg, MySQLdb and Django with parameters). The comment's URL-encoded values contain `%`, which must then be doubled. The Django and SQLAlchemy integrations do this for you.
|
|
62
|
+
|
|
63
|
+
## Notes
|
|
64
|
+
|
|
65
|
+
- **Traces:** an incoming `traceparent` header is continued, so traces line up with OpenTelemetry.
|
|
66
|
+
- **No raw URLs:** paths are never recorded, so ids and other values in them don't end up in your stats.
|
|
67
|
+
- **Existing comments:** SQL that already contains a comment is left unchanged, as the sqlcommenter spec requires.
|
|
68
|
+
- **Streaming responses:** in WSGI, queries made while a streamed response is iterated are not tagged.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
tests/test_sdk.py
|
|
4
|
+
tracelet_sdk/__init__.py
|
|
5
|
+
tracelet_sdk/asgi.py
|
|
6
|
+
tracelet_sdk/django.py
|
|
7
|
+
tracelet_sdk/flask.py
|
|
8
|
+
tracelet_sdk/sqlalchemy.py
|
|
9
|
+
tracelet_sdk/wsgi.py
|
|
10
|
+
tracelet_sdk.egg-info/PKG-INFO
|
|
11
|
+
tracelet_sdk.egg-info/SOURCES.txt
|
|
12
|
+
tracelet_sdk.egg-info/dependency_links.txt
|
|
13
|
+
tracelet_sdk.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
tracelet_sdk
|