toolscope-obs 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolscope/__init__.py +13 -0
- toolscope/config.py +19 -0
- toolscope/event_queue.py +49 -0
- toolscope/events.py +27 -0
- toolscope/heartbeat.py +42 -0
- toolscope/http_client.py +61 -0
- toolscope/instrumentation.py +241 -0
- toolscope/jobs.py +60 -0
- toolscope/observe.py +171 -0
- toolscope/session.py +27 -0
- toolscope/worker.py +46 -0
- toolscope_obs-0.1.0.dist-info/METADATA +64 -0
- toolscope_obs-0.1.0.dist-info/RECORD +15 -0
- toolscope_obs-0.1.0.dist-info/WHEEL +5 -0
- toolscope_obs-0.1.0.dist-info/top_level.txt +1 -0
toolscope/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Toolscope - Production observability for your applications.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
import toolscope
|
|
5
|
+
|
|
6
|
+
toolscope.init(api_key="ts_live_xxxxxxxxx")
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from . import observe
|
|
10
|
+
from .observe import init, flush, shutdown
|
|
11
|
+
from .jobs import job, metric, get_run_id, set_run_id
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0"
|
toolscope/config.py
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class Config:
|
|
7
|
+
api_key: str
|
|
8
|
+
backend_url: str = "https://api.toolscope.dev/v1"
|
|
9
|
+
max_queue_size: int = 10000
|
|
10
|
+
flush_interval_seconds: float = 5.0
|
|
11
|
+
heartbeat_interval_seconds: float = 30.0
|
|
12
|
+
max_retries: int = 3
|
|
13
|
+
debug: bool = False
|
|
14
|
+
|
|
15
|
+
def __post_init__(self):
|
|
16
|
+
if not self.api_key or not isinstance(self.api_key, str):
|
|
17
|
+
raise ValueError("api_key is required and must be a string")
|
|
18
|
+
if not self.api_key.startswith("ts_live_"):
|
|
19
|
+
raise ValueError("Invalid API key format. Must start with 'ts_live_'")
|
toolscope/event_queue.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import queue
|
|
2
|
+
import threading
|
|
3
|
+
from typing import List
|
|
4
|
+
|
|
5
|
+
from .config import Config
|
|
6
|
+
from .events import Event
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class EventQueue:
|
|
10
|
+
def __init__(self, config: Config):
|
|
11
|
+
self._config = config
|
|
12
|
+
self._queue: queue.Queue[Event] = queue.Queue(maxsize=config.max_queue_size)
|
|
13
|
+
self._dropped_count = 0
|
|
14
|
+
self._lock = threading.Lock()
|
|
15
|
+
|
|
16
|
+
def enqueue(self, event: Event) -> bool:
|
|
17
|
+
try:
|
|
18
|
+
self._queue.put_nowait(event)
|
|
19
|
+
return True
|
|
20
|
+
except queue.Full:
|
|
21
|
+
with self._lock:
|
|
22
|
+
self._dropped_count += 1
|
|
23
|
+
return False
|
|
24
|
+
|
|
25
|
+
def dequeue(self, batch_size: int = 50) -> List[Event]:
|
|
26
|
+
events: List[Event] = []
|
|
27
|
+
for _ in range(batch_size):
|
|
28
|
+
try:
|
|
29
|
+
events.append(self._queue.get_nowait())
|
|
30
|
+
except queue.Empty:
|
|
31
|
+
break
|
|
32
|
+
return events
|
|
33
|
+
|
|
34
|
+
def size(self) -> int:
|
|
35
|
+
return self._queue.qsize()
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def dropped_count(self) -> int:
|
|
39
|
+
with self._lock:
|
|
40
|
+
return self._dropped_count
|
|
41
|
+
|
|
42
|
+
def flush(self) -> List[Event]:
|
|
43
|
+
all_events: List[Event] = []
|
|
44
|
+
while True:
|
|
45
|
+
try:
|
|
46
|
+
all_events.append(self._queue.get_nowait())
|
|
47
|
+
except queue.Empty:
|
|
48
|
+
break
|
|
49
|
+
return all_events
|
toolscope/events.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
from typing import Any, Dict, Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Event:
|
|
7
|
+
def __init__(
|
|
8
|
+
self,
|
|
9
|
+
event_type: str,
|
|
10
|
+
data: Optional[Dict[str, Any]] = None,
|
|
11
|
+
session_id: Optional[str] = None,
|
|
12
|
+
):
|
|
13
|
+
self.type = event_type
|
|
14
|
+
self.timestamp = datetime.now(timezone.utc).isoformat()
|
|
15
|
+
self.session_id = session_id
|
|
16
|
+
self.data = data or {}
|
|
17
|
+
|
|
18
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
19
|
+
return {
|
|
20
|
+
"type": self.type,
|
|
21
|
+
"timestamp": self.timestamp,
|
|
22
|
+
"session_id": self.session_id,
|
|
23
|
+
"data": self.data,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
def to_json(self) -> str:
|
|
27
|
+
return json.dumps(self.to_dict())
|
toolscope/heartbeat.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from .config import Config
|
|
6
|
+
from .event_queue import EventQueue
|
|
7
|
+
from .events import Event
|
|
8
|
+
from .session import Session
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Heartbeat:
|
|
12
|
+
def __init__(self, queue: EventQueue, session: Session, config: Config):
|
|
13
|
+
self._queue = queue
|
|
14
|
+
self._session = session
|
|
15
|
+
self._config = config
|
|
16
|
+
self._thread: Optional[threading.Thread] = None
|
|
17
|
+
self._stop_event = threading.Event()
|
|
18
|
+
|
|
19
|
+
def start(self):
|
|
20
|
+
if self._thread is not None:
|
|
21
|
+
return
|
|
22
|
+
self._stop_event.clear()
|
|
23
|
+
self._thread = threading.Thread(target=self._run, daemon=True)
|
|
24
|
+
self._thread.start()
|
|
25
|
+
|
|
26
|
+
def stop(self):
|
|
27
|
+
self._stop_event.set()
|
|
28
|
+
if self._thread is not None:
|
|
29
|
+
self._thread.join(timeout=5.0)
|
|
30
|
+
self._thread = None
|
|
31
|
+
|
|
32
|
+
def _run(self):
|
|
33
|
+
while not self._stop_event.wait(self._config.heartbeat_interval_seconds):
|
|
34
|
+
event = Event(
|
|
35
|
+
event_type="heartbeat",
|
|
36
|
+
data={
|
|
37
|
+
"queue_size": self._queue.size(),
|
|
38
|
+
"dropped_events": self._queue.dropped_count,
|
|
39
|
+
},
|
|
40
|
+
session_id=self._session.session_id,
|
|
41
|
+
)
|
|
42
|
+
self._queue.enqueue(event)
|
toolscope/http_client.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import urllib.error
|
|
3
|
+
import urllib.request
|
|
4
|
+
from typing import Any, Dict, List, Optional
|
|
5
|
+
|
|
6
|
+
from .config import Config
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class HTTPClient:
|
|
10
|
+
def __init__(self, config: Config):
|
|
11
|
+
self._config = config
|
|
12
|
+
self._headers = {
|
|
13
|
+
"Content-Type": "application/json",
|
|
14
|
+
"Authorization": f"Bearer {config.api_key}",
|
|
15
|
+
"User-Agent": "toolscope-python/0.1.0",
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
def _request(
|
|
19
|
+
self,
|
|
20
|
+
method: str,
|
|
21
|
+
path: str,
|
|
22
|
+
body: Optional[Dict[str, Any]] = None,
|
|
23
|
+
timeout: float = 10.0,
|
|
24
|
+
) -> Optional[Dict[str, Any]]:
|
|
25
|
+
url = f"{self._config.backend_url.rstrip('/')}/{path.lstrip('/')}"
|
|
26
|
+
data = json.dumps(body).encode("utf-8") if body else None
|
|
27
|
+
|
|
28
|
+
req = urllib.request.Request(
|
|
29
|
+
url,
|
|
30
|
+
data=data,
|
|
31
|
+
headers=self._headers,
|
|
32
|
+
method=method,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
37
|
+
resp_body = resp.read().decode("utf-8")
|
|
38
|
+
return json.loads(resp_body) if resp_body else None
|
|
39
|
+
except urllib.error.HTTPError as e:
|
|
40
|
+
if e.code == 401 and self._config.debug:
|
|
41
|
+
print(f"[toolscope] Invalid API key: {e.reason}")
|
|
42
|
+
if self._config.debug:
|
|
43
|
+
print(f"[toolscope] HTTP error {e.code}: {e.reason}")
|
|
44
|
+
return None
|
|
45
|
+
except (urllib.error.URLError, OSError) as e:
|
|
46
|
+
if self._config.debug:
|
|
47
|
+
print(f"[toolscope] Connection error: {e}")
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
def validate_api_key(self) -> bool:
|
|
51
|
+
result = self._request("GET", "/validate")
|
|
52
|
+
return result is not None and result.get("valid", False)
|
|
53
|
+
|
|
54
|
+
def send_events(self, events: List[Dict[str, Any]]) -> bool:
|
|
55
|
+
result = self._request(
|
|
56
|
+
"POST",
|
|
57
|
+
"/events",
|
|
58
|
+
body={"events": events},
|
|
59
|
+
timeout=15.0,
|
|
60
|
+
)
|
|
61
|
+
return result is not None
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import atexit
|
|
2
|
+
import logging
|
|
3
|
+
import sys
|
|
4
|
+
import threading
|
|
5
|
+
import traceback
|
|
6
|
+
from typing import Any, List, Optional, TextIO
|
|
7
|
+
|
|
8
|
+
from .config import Config
|
|
9
|
+
from .event_queue import EventQueue
|
|
10
|
+
from .events import Event
|
|
11
|
+
from .session import Session
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class _StreamWrapper:
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
original: TextIO,
|
|
18
|
+
stream_name: str,
|
|
19
|
+
queue: EventQueue,
|
|
20
|
+
session_id: str,
|
|
21
|
+
):
|
|
22
|
+
self._original = original
|
|
23
|
+
self._stream_name = stream_name
|
|
24
|
+
self._queue = queue
|
|
25
|
+
self._session_id = session_id
|
|
26
|
+
|
|
27
|
+
def write(self, text: str):
|
|
28
|
+
stripped = text.rstrip("\n")
|
|
29
|
+
if stripped:
|
|
30
|
+
from .jobs import get_run_id
|
|
31
|
+
run_id = get_run_id()
|
|
32
|
+
data = {
|
|
33
|
+
"message": stripped,
|
|
34
|
+
"level": "info" if self._stream_name == "stdout" else "error",
|
|
35
|
+
"stream": self._stream_name,
|
|
36
|
+
}
|
|
37
|
+
if run_id:
|
|
38
|
+
data["run_id"] = run_id
|
|
39
|
+
event = Event(
|
|
40
|
+
event_type="log",
|
|
41
|
+
data=data,
|
|
42
|
+
session_id=self._session_id,
|
|
43
|
+
)
|
|
44
|
+
self._queue.enqueue(event)
|
|
45
|
+
return self._original.write(text)
|
|
46
|
+
|
|
47
|
+
def flush(self):
|
|
48
|
+
return self._original.flush()
|
|
49
|
+
|
|
50
|
+
def __getattr__(self, name: str):
|
|
51
|
+
return getattr(self._original, name)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class _LogHandler(logging.Handler):
|
|
55
|
+
def __init__(self, queue: EventQueue, session_id: str, level: int = logging.INFO):
|
|
56
|
+
super().__init__(level)
|
|
57
|
+
self._queue = queue
|
|
58
|
+
self._session_id = session_id
|
|
59
|
+
|
|
60
|
+
def emit(self, record: logging.LogRecord):
|
|
61
|
+
from .jobs import get_run_id
|
|
62
|
+
run_id = get_run_id()
|
|
63
|
+
data = {
|
|
64
|
+
"message": self.format(record),
|
|
65
|
+
"level": record.levelname.lower(),
|
|
66
|
+
"logger": record.name,
|
|
67
|
+
"stream": "logging",
|
|
68
|
+
}
|
|
69
|
+
if run_id:
|
|
70
|
+
data["run_id"] = run_id
|
|
71
|
+
event = Event(
|
|
72
|
+
event_type="log",
|
|
73
|
+
data=data,
|
|
74
|
+
session_id=self._session_id,
|
|
75
|
+
)
|
|
76
|
+
self._queue.enqueue(event)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class Instrumentation:
|
|
80
|
+
def __init__(self, queue: EventQueue, session: Session, config: Config):
|
|
81
|
+
self._queue = queue
|
|
82
|
+
self._session = session
|
|
83
|
+
self._config = config
|
|
84
|
+
self._original_stdout: Optional[TextIO] = None
|
|
85
|
+
self._original_stderr: Optional[TextIO] = None
|
|
86
|
+
self._original_excepthook: Optional[Any] = None
|
|
87
|
+
self._original_request: Optional[Any] = None
|
|
88
|
+
self._original_session_request: Optional[Any] = None
|
|
89
|
+
self._log_handler: Optional[_LogHandler] = None
|
|
90
|
+
self._patches: List[str] = []
|
|
91
|
+
|
|
92
|
+
def install(self):
|
|
93
|
+
self._patch_io_streams()
|
|
94
|
+
self._patch_excepthook()
|
|
95
|
+
self._patch_logging()
|
|
96
|
+
self._patch_requests()
|
|
97
|
+
|
|
98
|
+
def uninstall(self):
|
|
99
|
+
self._restore_io_streams()
|
|
100
|
+
self._restore_excepthook()
|
|
101
|
+
self._restore_logging()
|
|
102
|
+
self._restore_requests()
|
|
103
|
+
|
|
104
|
+
def _patch_io_streams(self):
|
|
105
|
+
self._original_stdout = sys.stdout
|
|
106
|
+
self._original_stderr = sys.stderr
|
|
107
|
+
|
|
108
|
+
if self._original_stdout is not None:
|
|
109
|
+
sys.stdout = _StreamWrapper(
|
|
110
|
+
self._original_stdout, "stdout", self._queue, self._session.session_id
|
|
111
|
+
)
|
|
112
|
+
if self._original_stderr is not None:
|
|
113
|
+
sys.stderr = _StreamWrapper(
|
|
114
|
+
self._original_stderr, "stderr", self._queue, self._session.session_id
|
|
115
|
+
)
|
|
116
|
+
self._patches.append("iostreams")
|
|
117
|
+
|
|
118
|
+
def _restore_io_streams(self):
|
|
119
|
+
if self._original_stdout is not None:
|
|
120
|
+
sys.stdout = self._original_stdout
|
|
121
|
+
self._original_stdout = None
|
|
122
|
+
if self._original_stderr is not None:
|
|
123
|
+
sys.stderr = self._original_stderr
|
|
124
|
+
self._original_stderr = None
|
|
125
|
+
|
|
126
|
+
def _patch_excepthook(self):
|
|
127
|
+
self._original_excepthook = sys.excepthook
|
|
128
|
+
queue = self._queue
|
|
129
|
+
session_id = self._session.session_id
|
|
130
|
+
|
|
131
|
+
def _patched_excepthook(exc_type, exc_value, exc_tb):
|
|
132
|
+
tb_lines = traceback.format_exception(exc_type, exc_value, exc_tb)
|
|
133
|
+
from .jobs import get_run_id
|
|
134
|
+
run_id = get_run_id()
|
|
135
|
+
data = {
|
|
136
|
+
"type": exc_type.__name__,
|
|
137
|
+
"message": str(exc_value),
|
|
138
|
+
"stacktrace": "".join(tb_lines),
|
|
139
|
+
}
|
|
140
|
+
if run_id:
|
|
141
|
+
data["run_id"] = run_id
|
|
142
|
+
event = Event(
|
|
143
|
+
event_type="exception",
|
|
144
|
+
data=data,
|
|
145
|
+
session_id=session_id,
|
|
146
|
+
)
|
|
147
|
+
queue.enqueue(event)
|
|
148
|
+
if self._original_excepthook is not None:
|
|
149
|
+
original_stderr = sys.stderr
|
|
150
|
+
if self._original_stderr is not None:
|
|
151
|
+
sys.stderr = self._original_stderr
|
|
152
|
+
try:
|
|
153
|
+
self._original_excepthook(exc_type, exc_value, exc_tb)
|
|
154
|
+
finally:
|
|
155
|
+
sys.stderr = original_stderr
|
|
156
|
+
|
|
157
|
+
sys.excepthook = _patched_excepthook
|
|
158
|
+
self._patches.append("excepthook")
|
|
159
|
+
|
|
160
|
+
def _restore_excepthook(self):
|
|
161
|
+
if self._original_excepthook is not None:
|
|
162
|
+
sys.excepthook = self._original_excepthook
|
|
163
|
+
self._original_excepthook = None
|
|
164
|
+
|
|
165
|
+
def _patch_logging(self):
|
|
166
|
+
self._log_handler = _LogHandler(self._queue, self._session.session_id)
|
|
167
|
+
formatter = logging.Formatter("%(message)s")
|
|
168
|
+
self._log_handler.setFormatter(formatter)
|
|
169
|
+
root_logger = logging.getLogger()
|
|
170
|
+
root_logger.addHandler(self._log_handler)
|
|
171
|
+
|
|
172
|
+
def _restore_logging(self):
|
|
173
|
+
if self._log_handler is not None:
|
|
174
|
+
root_logger = logging.getLogger()
|
|
175
|
+
root_logger.removeHandler(self._log_handler)
|
|
176
|
+
self._log_handler.close()
|
|
177
|
+
self._log_handler = None
|
|
178
|
+
|
|
179
|
+
def _patch_requests(self):
|
|
180
|
+
try:
|
|
181
|
+
import requests as _requests
|
|
182
|
+
except ImportError:
|
|
183
|
+
return
|
|
184
|
+
|
|
185
|
+
backend_url = self._config.backend_url
|
|
186
|
+
queue = self._queue
|
|
187
|
+
session_id = self._session.session_id
|
|
188
|
+
|
|
189
|
+
original_session_request = _requests.Session.request
|
|
190
|
+
self._original_session_request = original_session_request
|
|
191
|
+
|
|
192
|
+
def _patched_session_request(session_self, method, url, *args, **kwargs):
|
|
193
|
+
if url.startswith(backend_url):
|
|
194
|
+
return original_session_request(
|
|
195
|
+
session_self, method, url, *args, **kwargs
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
import time as _time
|
|
199
|
+
|
|
200
|
+
start = _time.time()
|
|
201
|
+
status_code = 0
|
|
202
|
+
try:
|
|
203
|
+
response = original_session_request(
|
|
204
|
+
session_self, method, url, *args, **kwargs
|
|
205
|
+
)
|
|
206
|
+
status_code = response.status_code
|
|
207
|
+
return response
|
|
208
|
+
except BaseException:
|
|
209
|
+
status_code = 0
|
|
210
|
+
raise
|
|
211
|
+
finally:
|
|
212
|
+
duration_ms = (_time.time() - start) * 1000
|
|
213
|
+
from .jobs import get_run_id
|
|
214
|
+
run_id = get_run_id()
|
|
215
|
+
data = {
|
|
216
|
+
"method": method.upper(),
|
|
217
|
+
"url": url,
|
|
218
|
+
"status_code": status_code,
|
|
219
|
+
"duration_ms": round(duration_ms, 2),
|
|
220
|
+
}
|
|
221
|
+
if run_id:
|
|
222
|
+
data["run_id"] = run_id
|
|
223
|
+
event = Event(
|
|
224
|
+
event_type="http.request",
|
|
225
|
+
data=data,
|
|
226
|
+
session_id=session_id,
|
|
227
|
+
)
|
|
228
|
+
queue.enqueue(event)
|
|
229
|
+
|
|
230
|
+
_requests.Session.request = _patched_session_request # type: ignore[assignment]
|
|
231
|
+
self._patches.append("requests")
|
|
232
|
+
|
|
233
|
+
def _restore_requests(self):
|
|
234
|
+
if self._original_session_request is not None:
|
|
235
|
+
try:
|
|
236
|
+
import requests as _requests
|
|
237
|
+
|
|
238
|
+
_requests.Session.request = self._original_session_request
|
|
239
|
+
except ImportError:
|
|
240
|
+
pass
|
|
241
|
+
self._original_session_request = None
|
toolscope/jobs.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
import threading
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
from typing import Any, Optional
|
|
5
|
+
from .events import Event
|
|
6
|
+
|
|
7
|
+
_thread_local = threading.local()
|
|
8
|
+
|
|
9
|
+
def get_run_id() -> Optional[str]:
|
|
10
|
+
return getattr(_thread_local, "run_id", None)
|
|
11
|
+
|
|
12
|
+
def set_run_id(run_id: Optional[str]) -> None:
|
|
13
|
+
_thread_local.run_id = run_id
|
|
14
|
+
|
|
15
|
+
@contextmanager
|
|
16
|
+
def job(name: str):
|
|
17
|
+
from .observe import _queue, _session, _initialized
|
|
18
|
+
|
|
19
|
+
run_id = str(uuid.uuid4())
|
|
20
|
+
previous_run_id = get_run_id()
|
|
21
|
+
set_run_id(run_id)
|
|
22
|
+
|
|
23
|
+
if _initialized and _queue is not None and _session is not None:
|
|
24
|
+
event = Event(
|
|
25
|
+
event_type="job",
|
|
26
|
+
data={"name": name, "status": "started", "run_id": run_id},
|
|
27
|
+
session_id=_session.session_id
|
|
28
|
+
)
|
|
29
|
+
_queue.enqueue(event)
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
yield run_id
|
|
33
|
+
if _initialized and _queue is not None and _session is not None:
|
|
34
|
+
event = Event(
|
|
35
|
+
event_type="job",
|
|
36
|
+
data={"name": name, "status": "success", "run_id": run_id},
|
|
37
|
+
session_id=_session.session_id
|
|
38
|
+
)
|
|
39
|
+
_queue.enqueue(event)
|
|
40
|
+
except Exception as e:
|
|
41
|
+
if _initialized and _queue is not None and _session is not None:
|
|
42
|
+
event = Event(
|
|
43
|
+
event_type="job",
|
|
44
|
+
data={"name": name, "status": "error", "message": str(e), "run_id": run_id},
|
|
45
|
+
session_id=_session.session_id
|
|
46
|
+
)
|
|
47
|
+
_queue.enqueue(event)
|
|
48
|
+
raise
|
|
49
|
+
finally:
|
|
50
|
+
set_run_id(previous_run_id)
|
|
51
|
+
|
|
52
|
+
def metric(name: str, value: Any) -> None:
|
|
53
|
+
from .observe import _queue, _session, _initialized
|
|
54
|
+
if _initialized and _queue is not None and _session is not None:
|
|
55
|
+
event = Event(
|
|
56
|
+
event_type="metric",
|
|
57
|
+
data={"name": name, "value": value, "run_id": get_run_id()},
|
|
58
|
+
session_id=_session.session_id
|
|
59
|
+
)
|
|
60
|
+
_queue.enqueue(event)
|
toolscope/observe.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Public API for Toolscope observability SDK.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
from toolscope import observe
|
|
5
|
+
|
|
6
|
+
observe.init(api_key="ts_live_xxxxxxxxx")
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import atexit
|
|
10
|
+
import logging
|
|
11
|
+
import signal
|
|
12
|
+
import sys
|
|
13
|
+
import threading
|
|
14
|
+
from typing import Any, Optional
|
|
15
|
+
|
|
16
|
+
from .config import Config
|
|
17
|
+
from .event_queue import EventQueue
|
|
18
|
+
from .events import Event
|
|
19
|
+
from .heartbeat import Heartbeat
|
|
20
|
+
from .http_client import HTTPClient
|
|
21
|
+
from .instrumentation import Instrumentation
|
|
22
|
+
from .session import Session
|
|
23
|
+
from .worker import Worker
|
|
24
|
+
|
|
25
|
+
_log = logging.getLogger(__name__)
|
|
26
|
+
|
|
27
|
+
_initialized = False
|
|
28
|
+
_config: Optional[Config] = None
|
|
29
|
+
_queue: Optional[EventQueue] = None
|
|
30
|
+
_worker: Optional[Worker] = None
|
|
31
|
+
_session: Optional[Session] = None
|
|
32
|
+
_http_client: Optional[HTTPClient] = None
|
|
33
|
+
_instrumentation: Optional[Instrumentation] = None
|
|
34
|
+
_heartbeat: Optional[Heartbeat] = None
|
|
35
|
+
_lock = threading.Lock()
|
|
36
|
+
_shutdown_complete = False
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def init(api_key: str, **options: Any) -> None:
|
|
40
|
+
global _initialized
|
|
41
|
+
|
|
42
|
+
with _lock:
|
|
43
|
+
if _initialized:
|
|
44
|
+
return
|
|
45
|
+
_initialize(api_key, options)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _initialize(api_key: str, options: dict) -> None:
|
|
49
|
+
global _initialized, _config, _queue, _worker, _session
|
|
50
|
+
global _http_client, _instrumentation, _heartbeat, _shutdown_complete
|
|
51
|
+
|
|
52
|
+
_config = Config(api_key=api_key, **options)
|
|
53
|
+
|
|
54
|
+
if _config.debug:
|
|
55
|
+
logging.basicConfig(level=logging.DEBUG)
|
|
56
|
+
_log.debug("toolscope initializing")
|
|
57
|
+
|
|
58
|
+
_session = Session(_config)
|
|
59
|
+
_queue = EventQueue(_config)
|
|
60
|
+
_http_client = HTTPClient(_config)
|
|
61
|
+
|
|
62
|
+
_worker = Worker(_queue, _http_client, _config)
|
|
63
|
+
_worker.start()
|
|
64
|
+
|
|
65
|
+
_instrumentation = Instrumentation(_queue, _session, _config)
|
|
66
|
+
_instrumentation.install()
|
|
67
|
+
|
|
68
|
+
_heartbeat = Heartbeat(_queue, _session, _config)
|
|
69
|
+
_heartbeat.start()
|
|
70
|
+
|
|
71
|
+
_queue.enqueue(_session.create_start_event())
|
|
72
|
+
|
|
73
|
+
_validate_api_key()
|
|
74
|
+
_register_shutdown()
|
|
75
|
+
|
|
76
|
+
_initialized = True
|
|
77
|
+
|
|
78
|
+
if _config.debug:
|
|
79
|
+
_log.debug("toolscope initialized (session=%s)", _session.session_id)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def flush() -> None:
|
|
83
|
+
global _queue, _http_client
|
|
84
|
+
if _queue is None or _http_client is None:
|
|
85
|
+
return
|
|
86
|
+
events = _queue.flush()
|
|
87
|
+
if events:
|
|
88
|
+
_http_client.send_events([e.to_dict() for e in events])
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def shutdown() -> None:
|
|
92
|
+
_do_shutdown()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _validate_api_key() -> None:
|
|
96
|
+
global _http_client
|
|
97
|
+
client = _http_client
|
|
98
|
+
if client is None:
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
t = threading.Thread(
|
|
102
|
+
target=_validate_api_key_task,
|
|
103
|
+
args=(client,),
|
|
104
|
+
daemon=True,
|
|
105
|
+
)
|
|
106
|
+
t.start()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _validate_api_key_task(client: HTTPClient) -> None:
|
|
110
|
+
try:
|
|
111
|
+
valid = client.validate_api_key()
|
|
112
|
+
if not valid:
|
|
113
|
+
print(
|
|
114
|
+
"[toolscope] Warning: API key validation failed. "
|
|
115
|
+
"Events will still be queued but may not be accepted.",
|
|
116
|
+
file=sys.stderr,
|
|
117
|
+
)
|
|
118
|
+
except Exception as exc:
|
|
119
|
+
print(
|
|
120
|
+
f"[toolscope] Warning: Could not validate API key: {exc}",
|
|
121
|
+
file=sys.stderr,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _do_shutdown() -> None:
|
|
126
|
+
global _initialized, _shutdown_complete, _heartbeat, _queue
|
|
127
|
+
global _session, _worker, _instrumentation
|
|
128
|
+
|
|
129
|
+
with _lock:
|
|
130
|
+
if _shutdown_complete:
|
|
131
|
+
return
|
|
132
|
+
_shutdown_complete = True
|
|
133
|
+
|
|
134
|
+
if _config is not None and _config.debug:
|
|
135
|
+
_log.debug("toolscope shutting down")
|
|
136
|
+
|
|
137
|
+
if _heartbeat is not None:
|
|
138
|
+
_heartbeat.stop()
|
|
139
|
+
|
|
140
|
+
if _session is not None and _queue is not None:
|
|
141
|
+
_queue.enqueue(
|
|
142
|
+
Event(
|
|
143
|
+
event_type="session.end",
|
|
144
|
+
data={},
|
|
145
|
+
session_id=_session.session_id,
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
flush()
|
|
150
|
+
|
|
151
|
+
if _worker is not None:
|
|
152
|
+
_worker.stop()
|
|
153
|
+
|
|
154
|
+
if _instrumentation is not None:
|
|
155
|
+
_instrumentation.uninstall()
|
|
156
|
+
|
|
157
|
+
_initialized = False
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _register_shutdown() -> None:
|
|
161
|
+
atexit.register(_do_shutdown)
|
|
162
|
+
|
|
163
|
+
try:
|
|
164
|
+
signal.signal(signal.SIGTERM, lambda *_: _do_shutdown())
|
|
165
|
+
except (ValueError, RuntimeError):
|
|
166
|
+
pass
|
|
167
|
+
|
|
168
|
+
try:
|
|
169
|
+
signal.signal(signal.SIGINT, lambda *_: _do_shutdown())
|
|
170
|
+
except (ValueError, RuntimeError):
|
|
171
|
+
pass
|
toolscope/session.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import platform
|
|
2
|
+
import sys
|
|
3
|
+
import uuid
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from typing import Any, Dict
|
|
6
|
+
|
|
7
|
+
from .config import Config
|
|
8
|
+
from .events import Event
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class Session:
|
|
12
|
+
def __init__(self, config: Config):
|
|
13
|
+
self.session_id = str(uuid.uuid4())
|
|
14
|
+
self.started_at = datetime.now(timezone.utc).isoformat()
|
|
15
|
+
self.config = config
|
|
16
|
+
|
|
17
|
+
def create_start_event(self) -> Event:
|
|
18
|
+
return Event(
|
|
19
|
+
event_type="session.start",
|
|
20
|
+
data={
|
|
21
|
+
"language": "python",
|
|
22
|
+
"language_version": sys.version,
|
|
23
|
+
"platform": platform.platform(),
|
|
24
|
+
"python_implementation": platform.python_implementation(),
|
|
25
|
+
},
|
|
26
|
+
session_id=self.session_id,
|
|
27
|
+
)
|
toolscope/worker.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
import time
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from .config import Config
|
|
6
|
+
from .event_queue import EventQueue
|
|
7
|
+
from .http_client import HTTPClient
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Worker:
|
|
11
|
+
def __init__(self, queue: EventQueue, http_client: HTTPClient, config: Config):
|
|
12
|
+
self._queue = queue
|
|
13
|
+
self._http_client = http_client
|
|
14
|
+
self._config = config
|
|
15
|
+
self._thread: Optional[threading.Thread] = None
|
|
16
|
+
self._stop_event = threading.Event()
|
|
17
|
+
|
|
18
|
+
def start(self):
|
|
19
|
+
if self._thread is not None:
|
|
20
|
+
return
|
|
21
|
+
self._stop_event.clear()
|
|
22
|
+
self._thread = threading.Thread(target=self._run, daemon=True)
|
|
23
|
+
self._thread.start()
|
|
24
|
+
|
|
25
|
+
def stop(self):
|
|
26
|
+
self._stop_event.set()
|
|
27
|
+
if self._thread is not None:
|
|
28
|
+
self._thread.join(timeout=5.0)
|
|
29
|
+
self._thread = None
|
|
30
|
+
|
|
31
|
+
def _run(self):
|
|
32
|
+
while not self._stop_event.is_set():
|
|
33
|
+
events = self._queue.dequeue(batch_size=50)
|
|
34
|
+
if events:
|
|
35
|
+
event_dicts = [e.to_dict() for e in events]
|
|
36
|
+
try:
|
|
37
|
+
success = self._http_client.send_events(event_dicts)
|
|
38
|
+
if not success and self._config.debug:
|
|
39
|
+
print(
|
|
40
|
+
f"[toolscope] Failed to send batch of {len(events)} events"
|
|
41
|
+
)
|
|
42
|
+
except Exception as e:
|
|
43
|
+
if self._config.debug:
|
|
44
|
+
print(f"[toolscope] Worker error: {e}")
|
|
45
|
+
else:
|
|
46
|
+
self._stop_event.wait(self._config.flush_interval_seconds)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: toolscope-obs
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Production observability for your applications
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: homepage, https://toolscope.dev
|
|
7
|
+
Project-URL: source, https://github.com/toolscope/toolscope-python
|
|
8
|
+
Keywords: observability,monitoring,logging,sdk
|
|
9
|
+
Requires-Python: >=3.8
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
Provides-Extra: requests
|
|
12
|
+
Requires-Dist: requests>=2.25.0; extra == "requests"
|
|
13
|
+
Provides-Extra: all
|
|
14
|
+
Requires-Dist: requests>=2.25.0; extra == "all"
|
|
15
|
+
|
|
16
|
+
# Toolscope Python SDK
|
|
17
|
+
|
|
18
|
+
Production observability and telemetry collection SDK for python applications and Wikimedia Toolforge tools.
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
You can install the Toolscope SDK from PyPI:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install toolscope-obs
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Or install with optional requests instrumentation support:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install "toolscope-obs[requests]"
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quick Start
|
|
35
|
+
|
|
36
|
+
Initialize the SDK at the entry point of your application:
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
import toolscope
|
|
40
|
+
import logging
|
|
41
|
+
|
|
42
|
+
# Initialize the SDK
|
|
43
|
+
toolscope.init(
|
|
44
|
+
api_key="ts_live_your_api_key",
|
|
45
|
+
backend_url="http://localhost:3000/api" # Optional: defaults to production api
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
# Start logging
|
|
49
|
+
logging.info("Application started")
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Job and Metric Monitoring
|
|
53
|
+
|
|
54
|
+
Isolate specific runs, jobs, or requests in your application using the `job` context manager:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
with toolscope.job("generate_image") as run_id:
|
|
58
|
+
# All print outputs, logs, requests, and unhandled errors
|
|
59
|
+
# inside this block are automatically tagged with the job's run_id.
|
|
60
|
+
print("Beginning image rendering task...")
|
|
61
|
+
|
|
62
|
+
# Track custom metrics
|
|
63
|
+
toolscope.metric("GPU_temp_c", 65.5)
|
|
64
|
+
```
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
toolscope/__init__.py,sha256=Agb2X4yzrrNVjNm838AwxlVeaP_z8Y4eu1PqC6amPMc,288
|
|
2
|
+
toolscope/config.py,sha256=MNrlhUKkci43ZxcWNZWAQ6Twj_Oi0Qv2y7fkFJCrQ6I,632
|
|
3
|
+
toolscope/event_queue.py,sha256=8RKnqN9omtvJqnmXn6cyl6LuQZDCSvaw5MAzIVvu4YM,1308
|
|
4
|
+
toolscope/events.py,sha256=h7BhT0TgBKtDLcykcQzn_nDaURyOKha2PhfCNd72KoA,710
|
|
5
|
+
toolscope/heartbeat.py,sha256=LFv3lIrrupm7dsFrH2bZSU0x03TJMxrRUrqayUqHp9s,1275
|
|
6
|
+
toolscope/http_client.py,sha256=EsSezDmlis8AwGDtkmVjEcvV1Q1K7heV7stz-dIcKWE,1967
|
|
7
|
+
toolscope/instrumentation.py,sha256=uGZObSbzO31ogIOh5iKhG6H8rjML64cqgaLs9wNlSuA,7994
|
|
8
|
+
toolscope/jobs.py,sha256=YUQhTqRhHllM6TNeWvgluHaPt24Rjp6mSv3U_aVwk-8,1935
|
|
9
|
+
toolscope/observe.py,sha256=U_Ci-Oo6ECNCLxVl4w1OqLiYk8EwrR86UxojB0Q6fRQ,4125
|
|
10
|
+
toolscope/session.py,sha256=COyChQ3N3jwZCuIOo6vQTROGvBWf9BQl8C8_mmOJ-Mw,758
|
|
11
|
+
toolscope/worker.py,sha256=ChjPm--JgcAB8DMcuPkRVP8WMSlIAKHbbbet6N6tVkQ,1573
|
|
12
|
+
toolscope_obs-0.1.0.dist-info/METADATA,sha256=E6t23z8h8pAjz-a3k43pmhHSopH3ya6PdjtZ4OYU474,1619
|
|
13
|
+
toolscope_obs-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
14
|
+
toolscope_obs-0.1.0.dist-info/top_level.txt,sha256=7g_4ZM9jR0OfZeYLGCN31iU-SfSTPc5D8ncZKWgHM6E,10
|
|
15
|
+
toolscope_obs-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
toolscope
|