linearjeph 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- linearjeph/__init__.py +13 -0
- linearjeph/client.py +632 -0
- linearjeph/health.py +134 -0
- linearjeph/integrations/__init__.py +0 -0
- linearjeph/integrations/asgi.py +41 -0
- linearjeph/integrations/django.py +55 -0
- linearjeph/integrations/flask.py +38 -0
- linearjeph/prophecy/__init__.py +519 -0
- linearjeph/prophecy/__main__.py +67 -0
- linearjeph/prophecy/django.py +63 -0
- linearjeph/prophecy/middleware.py +151 -0
- linearjeph/prophecy/scanner.py +507 -0
- linearjeph/py.typed +0 -0
- linearjeph/telemetry.py +452 -0
- linearjeph-1.2.0.dist-info/METADATA +107 -0
- linearjeph-1.2.0.dist-info/RECORD +19 -0
- linearjeph-1.2.0.dist-info/WHEEL +5 -0
- linearjeph-1.2.0.dist-info/licenses/LICENSE +21 -0
- linearjeph-1.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,519 @@
|
|
|
1
|
+
"""LINEARJEPH Prophecy: growth-intelligence tracking, codebase scan, history import and report access.
|
|
2
|
+
|
|
3
|
+
Standard library only. Tracking calls never raise into the host app (failures are swallowed and logged once), events go
|
|
4
|
+
through a bounded in-memory queue (10,000) to one background daemon thread, batched every 2 s or 100 events, gzip
|
|
5
|
+
compressed, retried with backoff on 429/5xx and flushed at interpreter exit. Secret-key calls are server side only.
|
|
6
|
+
"""
|
|
7
|
+
import atexit
|
|
8
|
+
import collections
|
|
9
|
+
import functools
|
|
10
|
+
import gzip
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import logging
|
|
14
|
+
import os
|
|
15
|
+
import random
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
import urllib.error
|
|
19
|
+
import urllib.request
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
|
|
22
|
+
from .scanner import scan as _scan
|
|
23
|
+
|
|
24
|
+
__all__ = ["init", "track", "feature", "signup", "payment", "subscription", "churn", "identify", "scan", "send_manifest",
|
|
25
|
+
"import_history", "report", "refresh", "live", "embed_url", "flush", "close", "normalize_path", "ProphecyError",
|
|
26
|
+
"SDK_VERSION"]
|
|
27
|
+
|
|
28
|
+
SDK_VERSION = "1.2.0"
|
|
29
|
+
DEFAULT_ENDPOINT = "https://linearjeph.cc"
|
|
30
|
+
P_EVENTS = "/ingest/v1/prophecy/events"
|
|
31
|
+
P_IDENTIFY = "/ingest/v1/prophecy/identify"
|
|
32
|
+
P_MANIFEST = "/ingest/v1/prophecy/manifest"
|
|
33
|
+
A_IMPORT = "/api/v1/prophecy/sdk/import"
|
|
34
|
+
A_REPORT = "/api/v1/prophecy/sdk/report"
|
|
35
|
+
A_REFRESH = "/api/v1/prophecy/sdk/refresh"
|
|
36
|
+
A_LIVE = "/api/v1/prophecy/sdk/live"
|
|
37
|
+
A_EMBED = "/api/v1/prophecy/sdk/embed"
|
|
38
|
+
MAX_QUEUE = 10000
|
|
39
|
+
BATCH = 100
|
|
40
|
+
INTERVAL = 2.0
|
|
41
|
+
IMPORT_BATCH = 5000
|
|
42
|
+
IMPORT_BYTES = int(4.5 * 1024 * 1024)
|
|
43
|
+
MANIFEST_BYTES = 500 * 1024
|
|
44
|
+
KINDS = {"signup", "login", "active", "feature", "payment", "refund", "subscription", "churn", "pageview", "request", "error", "custom"}
|
|
45
|
+
RETRY_STATUS = (429, 500, 502, 503, 504)
|
|
46
|
+
|
|
47
|
+
log = logging.getLogger("linearjeph.prophecy")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ProphecyError(RuntimeError):
|
|
51
|
+
"""Raised by the secret-key calls (import, report, refresh, live, embed_url) when the request cannot be completed."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, message, status=None):
|
|
54
|
+
super().__init__(message)
|
|
55
|
+
self.status = status
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_cond = threading.Condition()
|
|
59
|
+
_st = {"ingest_key": None, "secret_key": None, "endpoint": DEFAULT_ENDPOINT, "hash": True, "queue": collections.deque(), "thread": None,
|
|
60
|
+
"busy": False, "stop": False, "disabled": False, "dropped": 0, "retries": 4, "backoff": 0.5, "timeout": 10.0, "batch": BATCH,
|
|
61
|
+
"interval": INTERVAL, "warned": set(), "atexit": False}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _warn_once(msg):
|
|
65
|
+
if msg in _st["warned"]:
|
|
66
|
+
return
|
|
67
|
+
_st["warned"].add(msg)
|
|
68
|
+
try:
|
|
69
|
+
log.warning("linearjeph.prophecy: %s", msg)
|
|
70
|
+
except Exception: # noqa: BLE001
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _safe(fn):
|
|
75
|
+
"""Public tracking wrapper: swallow anything (log once) so the SDK can never break the host application."""
|
|
76
|
+
@functools.wraps(fn)
|
|
77
|
+
def wrapper(*a, **kw):
|
|
78
|
+
try:
|
|
79
|
+
return fn(*a, **kw)
|
|
80
|
+
except Exception as e: # noqa: BLE001 - by design
|
|
81
|
+
_warn_once("%s failed: %s" % (fn.__name__, e))
|
|
82
|
+
return None
|
|
83
|
+
return wrapper
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _sha256(s):
|
|
87
|
+
return hashlib.sha256(str(s).encode("utf-8")).hexdigest()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _uid(uid):
|
|
91
|
+
if uid is None or uid == "":
|
|
92
|
+
return ""
|
|
93
|
+
return _sha256(uid) if _st["hash"] else str(uid)[:200]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _iso(ts=None):
|
|
97
|
+
if ts is None:
|
|
98
|
+
return datetime.now(timezone.utc).isoformat()
|
|
99
|
+
if isinstance(ts, datetime):
|
|
100
|
+
return (ts if ts.tzinfo else ts.replace(tzinfo=timezone.utc)).isoformat()
|
|
101
|
+
if isinstance(ts, (int, float)):
|
|
102
|
+
return datetime.fromtimestamp(ts / 1000.0 if ts > 1e11 else ts, timezone.utc).isoformat()
|
|
103
|
+
return str(ts)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def init(ingest_key=None, secret_key=None, endpoint=None, hash_user_ids=True, batch_size=BATCH, flush_interval=INTERVAL, retries=4,
|
|
107
|
+
backoff=0.5, timeout=10.0, hooks=True):
|
|
108
|
+
"""Configure Prophecy. Keys fall back to LINEARJEPH_INGEST_KEY / LINEARJEPH_SECRET_KEY, endpoint to LINEARJEPH_ENDPOINT.
|
|
109
|
+
Returns True when at least one key is set. Never raises."""
|
|
110
|
+
try:
|
|
111
|
+
env = os.environ
|
|
112
|
+
ingest_key = ingest_key or env.get("LINEARJEPH_INGEST_KEY") or None
|
|
113
|
+
secret_key = secret_key or env.get("LINEARJEPH_SECRET_KEY") or None
|
|
114
|
+
if ingest_key and str(ingest_key).startswith("ljp_sk_"):
|
|
115
|
+
log.error("linearjeph.prophecy: ingest_key looks like a secret key (ljp_sk_...); use the application ingest key. Tracking disabled.")
|
|
116
|
+
return False
|
|
117
|
+
with _cond:
|
|
118
|
+
_st["queue"].clear()
|
|
119
|
+
_st.update(ingest_key=str(ingest_key) if ingest_key else None, secret_key=str(secret_key) if secret_key else None,
|
|
120
|
+
endpoint=str(endpoint or env.get("LINEARJEPH_ENDPOINT") or DEFAULT_ENDPOINT).rstrip("/"),
|
|
121
|
+
hash=bool(hash_user_ids), stop=False, batch=max(1, min(int(batch_size), 500)), interval=max(0.05, float(flush_interval)),
|
|
122
|
+
retries=max(0, int(retries)), backoff=float(backoff), timeout=float(timeout), disabled=False, dropped=0)
|
|
123
|
+
_st["warned"].clear()
|
|
124
|
+
if _st["ingest_key"]:
|
|
125
|
+
_ensure_thread()
|
|
126
|
+
if hooks and not _st["atexit"]:
|
|
127
|
+
_st["atexit"] = True
|
|
128
|
+
atexit.register(_at_exit)
|
|
129
|
+
return bool(_st["ingest_key"] or _st["secret_key"])
|
|
130
|
+
except Exception: # noqa: BLE001
|
|
131
|
+
return False
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ------------------------------------------------------------------ transport
|
|
135
|
+
def _post_ingest(path, body, last_chance=False):
|
|
136
|
+
data = gzip.compress(json.dumps(body, separators=(",", ":")).encode("utf-8"))
|
|
137
|
+
retries = 0 if last_chance else _st["retries"]
|
|
138
|
+
for i in range(retries + 1):
|
|
139
|
+
wait = None
|
|
140
|
+
req = urllib.request.Request(_st["endpoint"] + path, data=data, method="POST", headers={
|
|
141
|
+
"Content-Type": "application/json", "Content-Encoding": "gzip", "X-LJ-Key": _st["ingest_key"] or "",
|
|
142
|
+
"User-Agent": "linearjeph-python/" + SDK_VERSION})
|
|
143
|
+
try:
|
|
144
|
+
with urllib.request.urlopen(req, timeout=_st["timeout"]) as r:
|
|
145
|
+
r.read()
|
|
146
|
+
return True
|
|
147
|
+
except urllib.error.HTTPError as e:
|
|
148
|
+
code = e.code
|
|
149
|
+
if code == 401:
|
|
150
|
+
_st["disabled"] = True
|
|
151
|
+
_warn_once("ingest key rejected (401); tracking disabled")
|
|
152
|
+
return False
|
|
153
|
+
if code == 402:
|
|
154
|
+
_warn_once("Prophecy subscription inactive (402); events dropped")
|
|
155
|
+
return False
|
|
156
|
+
if code not in RETRY_STATUS:
|
|
157
|
+
_warn_once("%s answered %s; batch dropped" % (path, code))
|
|
158
|
+
return False
|
|
159
|
+
if code == 429:
|
|
160
|
+
try:
|
|
161
|
+
ra = float(e.headers.get("Retry-After") or 0)
|
|
162
|
+
if ra > 0:
|
|
163
|
+
wait = min(ra, 60.0)
|
|
164
|
+
except (TypeError, ValueError):
|
|
165
|
+
pass
|
|
166
|
+
except Exception: # noqa: BLE001 - network error or timeout: retry
|
|
167
|
+
pass
|
|
168
|
+
if i < retries:
|
|
169
|
+
time.sleep(wait if wait is not None else _st["backoff"] * (2 ** i) * (0.5 + random.random() / 2))
|
|
170
|
+
_warn_once("gave up sending to " + path)
|
|
171
|
+
return False
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _take_batch():
|
|
175
|
+
q = _st["queue"]
|
|
176
|
+
if not q:
|
|
177
|
+
return None, []
|
|
178
|
+
path = q[0][0]
|
|
179
|
+
batch, keep = [], collections.deque()
|
|
180
|
+
while q:
|
|
181
|
+
item = q.popleft()
|
|
182
|
+
if item[0] == path and len(batch) < _st["batch"]:
|
|
183
|
+
batch.append(item[1])
|
|
184
|
+
else:
|
|
185
|
+
keep.append(item)
|
|
186
|
+
q.extend(keep)
|
|
187
|
+
return path, batch
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _worker():
|
|
191
|
+
while True:
|
|
192
|
+
with _cond:
|
|
193
|
+
if _st["stop"] and not _st["queue"]:
|
|
194
|
+
return
|
|
195
|
+
if not _st["queue"] and not _st["stop"]:
|
|
196
|
+
_cond.wait(_st["interval"])
|
|
197
|
+
elif _st["queue"] and len(_st["queue"]) < _st["batch"] and not _st["stop"] and not _st.get("flushing"):
|
|
198
|
+
_cond.wait(_st["interval"])
|
|
199
|
+
if _st["disabled"]:
|
|
200
|
+
_st["queue"].clear()
|
|
201
|
+
_cond.notify_all()
|
|
202
|
+
continue
|
|
203
|
+
path, batch = _take_batch()
|
|
204
|
+
if batch:
|
|
205
|
+
_st["busy"] = True
|
|
206
|
+
if batch:
|
|
207
|
+
try:
|
|
208
|
+
_post_ingest(path, {("events" if path == P_EVENTS else "customers"): batch}, last_chance=_st["stop"])
|
|
209
|
+
except Exception as e: # noqa: BLE001
|
|
210
|
+
_warn_once("send error: %s" % e)
|
|
211
|
+
finally:
|
|
212
|
+
with _cond:
|
|
213
|
+
_st["busy"] = False
|
|
214
|
+
_cond.notify_all()
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _ensure_thread():
|
|
218
|
+
with _cond:
|
|
219
|
+
t = _st["thread"]
|
|
220
|
+
if t is not None and t.is_alive():
|
|
221
|
+
return
|
|
222
|
+
_st["stop"] = False
|
|
223
|
+
t = threading.Thread(target=_worker, name="linearjeph-prophecy", daemon=True)
|
|
224
|
+
_st["thread"] = t
|
|
225
|
+
t.start()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _enqueue(path, item):
|
|
229
|
+
if not _st["ingest_key"] or _st["disabled"]:
|
|
230
|
+
return
|
|
231
|
+
with _cond:
|
|
232
|
+
if len(_st["queue"]) >= MAX_QUEUE:
|
|
233
|
+
_st["dropped"] += 1
|
|
234
|
+
return
|
|
235
|
+
_st["queue"].append((path, item))
|
|
236
|
+
if len(_st["queue"]) >= _st["batch"]:
|
|
237
|
+
_cond.notify_all()
|
|
238
|
+
t = _st["thread"]
|
|
239
|
+
if t is None or not t.is_alive():
|
|
240
|
+
_ensure_thread()
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def flush(timeout=5.0):
|
|
244
|
+
"""Wait until everything queued is delivered (or timeout seconds). Returns True when empty."""
|
|
245
|
+
end = time.time() + timeout
|
|
246
|
+
with _cond:
|
|
247
|
+
_st["flushing"] = True
|
|
248
|
+
try:
|
|
249
|
+
while (_st["queue"] or _st["busy"]) and not _st["disabled"]:
|
|
250
|
+
left = end - time.time()
|
|
251
|
+
if left <= 0:
|
|
252
|
+
break
|
|
253
|
+
_cond.notify_all()
|
|
254
|
+
_cond.wait(min(left, 0.05))
|
|
255
|
+
return _st["disabled"] or not (_st["queue"] or _st["busy"])
|
|
256
|
+
finally:
|
|
257
|
+
_st["flushing"] = False
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def close(timeout=5.0):
|
|
261
|
+
"""Flush, then stop the background thread."""
|
|
262
|
+
ok = flush(timeout)
|
|
263
|
+
with _cond:
|
|
264
|
+
_st["stop"] = True
|
|
265
|
+
_cond.notify_all()
|
|
266
|
+
t = _st["thread"]
|
|
267
|
+
if t is not None and t.is_alive() and t is not threading.current_thread():
|
|
268
|
+
t.join(min(timeout, 2.0))
|
|
269
|
+
return ok
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _at_exit():
|
|
273
|
+
try:
|
|
274
|
+
flush(5.0)
|
|
275
|
+
except Exception: # noqa: BLE001
|
|
276
|
+
pass
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
# ------------------------------------------------------------------ tracking
|
|
280
|
+
def _props(props):
|
|
281
|
+
if not isinstance(props, dict):
|
|
282
|
+
return None
|
|
283
|
+
try:
|
|
284
|
+
s = json.dumps(props, default=str)
|
|
285
|
+
except (TypeError, ValueError):
|
|
286
|
+
return None
|
|
287
|
+
if len(s) > 2048:
|
|
288
|
+
_warn_once("props over 2 KB dropped")
|
|
289
|
+
return None
|
|
290
|
+
return json.loads(s)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _event(name, uid=None, kind="custom", feature=None, amount=None, currency=None, props=None, ts=None):
|
|
294
|
+
ev = {"ts": _iso(ts), "name": str(name)[:80], "kind": kind if kind in KINDS else "custom"}
|
|
295
|
+
u = _uid(uid)
|
|
296
|
+
if u:
|
|
297
|
+
ev["uid"] = u
|
|
298
|
+
if feature:
|
|
299
|
+
ev["feature"] = str(feature)[:80]
|
|
300
|
+
if amount is not None:
|
|
301
|
+
ev["amount"] = float(amount)
|
|
302
|
+
if currency:
|
|
303
|
+
ev["currency"] = str(currency)[:3].upper()
|
|
304
|
+
p = _props(props)
|
|
305
|
+
if p:
|
|
306
|
+
ev["props"] = p
|
|
307
|
+
return ev
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _strip(d):
|
|
311
|
+
return {k: v for k, v in d.items() if v is not None and v != ""}
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@_safe
|
|
315
|
+
def track(name, uid=None, kind="custom", feature=None, amount=None, currency=None, props=None, ts=None):
|
|
316
|
+
"""Record one event. kind: signup, login, active, feature, payment, refund, subscription, churn, pageview, request, error, custom."""
|
|
317
|
+
_enqueue(P_EVENTS, _event(name, uid, kind, feature, amount, currency, props, ts))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@_safe
|
|
321
|
+
def feature(name, uid=None):
|
|
322
|
+
"""Shorthand for a feature-use event (kind=feature)."""
|
|
323
|
+
_enqueue(P_EVENTS, _event(name, uid, "feature", feature=name))
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
@_safe
|
|
327
|
+
def identify(uid, traits=None, **kw):
|
|
328
|
+
"""Set customer traits: signup_at, plan, mrr, country, channel; every other key goes to traits (max 20 keys, 2 KB)."""
|
|
329
|
+
t = dict(traits or {})
|
|
330
|
+
t.update(kw)
|
|
331
|
+
c = _strip({"uid": _uid(uid), "signup_at": t.pop("signup_at", None) or t.pop("signupAt", None), "plan": t.pop("plan", None),
|
|
332
|
+
"mrr": t.pop("mrr", None), "country": t.pop("country", None), "channel": t.pop("channel", None)})
|
|
333
|
+
t.pop("signupAt", None)
|
|
334
|
+
extra = _props(dict(list(t.items())[:20]))
|
|
335
|
+
if extra:
|
|
336
|
+
c["traits"] = extra
|
|
337
|
+
if c.get("uid"):
|
|
338
|
+
_enqueue(P_IDENTIFY, c)
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
@_safe
|
|
342
|
+
def signup(uid, plan=None, channel=None, country=None, ts=None):
|
|
343
|
+
_enqueue(P_EVENTS, _event("signup", uid, "signup", props=_strip({"plan": plan, "channel": channel, "country": country}), ts=ts))
|
|
344
|
+
identify(uid, signup_at=_iso(ts), plan=plan, channel=channel, country=country)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
@_safe
|
|
348
|
+
def payment(uid, amount, currency="USD", plan=None, ts=None):
|
|
349
|
+
"""A payment (amount > 0) or a refund (amount < 0)."""
|
|
350
|
+
a = float(amount)
|
|
351
|
+
k = "refund" if a < 0 else "payment"
|
|
352
|
+
_enqueue(P_EVENTS, _event(k, uid, k, amount=a, currency=currency or "USD", props=_strip({"plan": plan}), ts=ts))
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
@_safe
|
|
356
|
+
def subscription(uid, plan, mrr, status="active", ts=None):
|
|
357
|
+
"""status: active, cancelled or paused."""
|
|
358
|
+
_enqueue(P_EVENTS, _event("subscription", uid, "subscription", props=_strip({"plan": plan, "mrr": float(mrr), "status": status}), ts=ts))
|
|
359
|
+
identify(uid, plan=plan, mrr=mrr)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
@_safe
|
|
363
|
+
def churn(uid, reason=None, ts=None):
|
|
364
|
+
_enqueue(P_EVENTS, _event("churn", uid, "churn", props=_strip({"reason": str(reason)[:200] if reason else None}), ts=ts))
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
# ------------------------------------------------------------------ requests (used by the middlewares)
|
|
368
|
+
from .middleware import normalize_path # noqa: E402 (re-exported)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def record_request(route, method, status, duration_ms, uid=None):
|
|
372
|
+
"""Record one request: route template, method, status, duration only. Never bodies, headers or query strings."""
|
|
373
|
+
try:
|
|
374
|
+
_enqueue(P_EVENTS, _event("request", uid, "request", props={"route": str(route)[:200], "method": str(method)[:10],
|
|
375
|
+
"status": int(status), "duration_ms": round(float(duration_ms), 1)}))
|
|
376
|
+
except Exception: # noqa: BLE001
|
|
377
|
+
pass
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# ------------------------------------------------------------------ scan + manifest
|
|
381
|
+
def scan(root_dir=".", max_files=None):
|
|
382
|
+
"""Scan root_dir locally and return the manifest dict (names and counts only; see docs/sdk/prophecy.md)."""
|
|
383
|
+
return _scan(root_dir, max_files) if max_files else _scan(root_dir)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _fit_manifest(m):
|
|
387
|
+
out = dict(m)
|
|
388
|
+
for _ in range(40):
|
|
389
|
+
if len(json.dumps({"manifest": out})) <= MANIFEST_BYTES:
|
|
390
|
+
break
|
|
391
|
+
k = max(("routes", "dependencies", "features", "env_names"), key=lambda x: len(json.dumps(out.get(x) or [])))
|
|
392
|
+
out[k] = (out.get(k) or [])[:len(out.get(k) or []) // 2]
|
|
393
|
+
return out
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def send_manifest(manifest):
|
|
397
|
+
"""POST the manifest with the ingest key. Returns True on success, False otherwise (never raises)."""
|
|
398
|
+
try:
|
|
399
|
+
if not _st["ingest_key"]:
|
|
400
|
+
_warn_once("send_manifest needs init(ingest_key=...)")
|
|
401
|
+
return False
|
|
402
|
+
return _post_ingest(P_MANIFEST, {"manifest": _fit_manifest(manifest)})
|
|
403
|
+
except Exception as e: # noqa: BLE001
|
|
404
|
+
_warn_once("send_manifest failed: %s" % e)
|
|
405
|
+
return False
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
# ------------------------------------------------------------------ secret-key API (server side only)
|
|
409
|
+
def _api(method, path, body=None):
|
|
410
|
+
if not _st["secret_key"]:
|
|
411
|
+
raise ProphecyError("Prophecy secret_key missing: init(secret_key=...) or set LINEARJEPH_SECRET_KEY")
|
|
412
|
+
data = json.dumps(body, separators=(",", ":")).encode("utf-8") if body is not None else None
|
|
413
|
+
last = None
|
|
414
|
+
for i in range(_st["retries"] + 1):
|
|
415
|
+
wait = None
|
|
416
|
+
req = urllib.request.Request(_st["endpoint"] + path, data=data, method=method, headers={
|
|
417
|
+
"Authorization": "Bearer " + _st["secret_key"], "Content-Type": "application/json", "User-Agent": "linearjeph-python/" + SDK_VERSION})
|
|
418
|
+
try:
|
|
419
|
+
with urllib.request.urlopen(req, timeout=max(_st["timeout"], 30.0)) as r:
|
|
420
|
+
txt = r.read().decode("utf-8") or "null"
|
|
421
|
+
return json.loads(txt)
|
|
422
|
+
except urllib.error.HTTPError as e:
|
|
423
|
+
if e.code in RETRY_STATUS:
|
|
424
|
+
last = ProphecyError("Prophecy %s answered %s" % (path, e.code), e.code)
|
|
425
|
+
if e.code == 429:
|
|
426
|
+
try:
|
|
427
|
+
ra = float(e.headers.get("Retry-After") or 0)
|
|
428
|
+
wait = min(ra, 60.0) if ra > 0 else None
|
|
429
|
+
except (TypeError, ValueError):
|
|
430
|
+
pass
|
|
431
|
+
else:
|
|
432
|
+
detail = ""
|
|
433
|
+
try:
|
|
434
|
+
j = json.loads(e.read().decode("utf-8"))
|
|
435
|
+
detail = ": %s" % (j.get("detail") or j.get("error") or "")
|
|
436
|
+
except Exception: # noqa: BLE001
|
|
437
|
+
pass
|
|
438
|
+
raise ProphecyError("Prophecy %s answered %s%s" % (path, e.code, detail), e.code)
|
|
439
|
+
except (urllib.error.URLError, OSError) as e:
|
|
440
|
+
last = ProphecyError("Prophecy request failed: %s" % e)
|
|
441
|
+
if i < _st["retries"]:
|
|
442
|
+
time.sleep(wait if wait is not None else _st["backoff"] * (2 ** i) * (0.5 + random.random() / 2))
|
|
443
|
+
raise last or ProphecyError("Prophecy request failed")
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def _import_row(kind, r):
|
|
447
|
+
row = dict(r)
|
|
448
|
+
if row.get("uid") not in (None, ""):
|
|
449
|
+
row["uid"] = _uid(row["uid"])
|
|
450
|
+
for k in ("amount", "mrr"):
|
|
451
|
+
if isinstance(row.get(k), str) and row[k].strip() != "":
|
|
452
|
+
try:
|
|
453
|
+
row[k] = float(row[k])
|
|
454
|
+
except ValueError:
|
|
455
|
+
pass
|
|
456
|
+
row = {k: v for k, v in row.items() if v is not None and v != ""}
|
|
457
|
+
if kind == "events" and isinstance(row.get("props"), str):
|
|
458
|
+
try:
|
|
459
|
+
row["props"] = json.loads(row["props"])
|
|
460
|
+
except ValueError:
|
|
461
|
+
row.pop("props")
|
|
462
|
+
return row
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def import_history(kind, rows, on_progress=None):
|
|
466
|
+
"""Backfill history with the secret key. events rows: ts, uid, name, kind, feature, amount, currency, props, id.
|
|
467
|
+
customers rows: uid, signup_at, plan, mrr, country, channel. Batches of 5,000, retries on 429/5xx.
|
|
468
|
+
Returns {accepted, rejected, errors (first 20), jobs}."""
|
|
469
|
+
if kind not in ("events", "customers"):
|
|
470
|
+
raise ProphecyError('import_history kind must be "events" or "customers"')
|
|
471
|
+
if not _st["secret_key"]:
|
|
472
|
+
raise ProphecyError("Prophecy secret_key missing: init(secret_key=...) or set LINEARJEPH_SECRET_KEY")
|
|
473
|
+
rows = list(rows)
|
|
474
|
+
total = len(rows)
|
|
475
|
+
res = {"accepted": 0, "rejected": 0, "errors": [], "jobs": []}
|
|
476
|
+
i = 0
|
|
477
|
+
while i < total:
|
|
478
|
+
n = min(IMPORT_BATCH, total - i)
|
|
479
|
+
chunk = [_import_row(kind, r) for r in rows[i:i + n]]
|
|
480
|
+
while n > 1 and len(json.dumps({"kind": kind, "rows": chunk})) > IMPORT_BYTES:
|
|
481
|
+
n = (n + 1) // 2
|
|
482
|
+
chunk = chunk[:n]
|
|
483
|
+
r = _api("POST", A_IMPORT, {"kind": kind, "rows": chunk}) or {}
|
|
484
|
+
res["accepted"] += int(r.get("accepted") or 0)
|
|
485
|
+
res["rejected"] += int(r.get("rejected") or 0)
|
|
486
|
+
if r.get("job") is not None:
|
|
487
|
+
res["jobs"].append(r["job"])
|
|
488
|
+
for e in r.get("errors") or []:
|
|
489
|
+
if len(res["errors"]) < 20:
|
|
490
|
+
res["errors"].append(e)
|
|
491
|
+
i += n
|
|
492
|
+
if on_progress:
|
|
493
|
+
try:
|
|
494
|
+
on_progress({"done": i, "total": total, "accepted": res["accepted"], "rejected": res["rejected"]})
|
|
495
|
+
except Exception: # noqa: BLE001
|
|
496
|
+
pass
|
|
497
|
+
return res
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def report(section=None):
|
|
501
|
+
"""Latest report: {"report": {...}, "created_at", "stale"}; section="revenue" returns one section."""
|
|
502
|
+
from urllib.parse import quote
|
|
503
|
+
return _api("GET", A_REPORT + ("?section=" + quote(section) if section else ""))
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def refresh():
|
|
507
|
+
"""Queue a new report (limited per hour)."""
|
|
508
|
+
return _api("POST", A_REFRESH, {})
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def live():
|
|
512
|
+
"""Realtime counters."""
|
|
513
|
+
return _api("GET", A_LIVE)
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def embed_url(ttl=3600):
|
|
517
|
+
"""Signed URL to show the report in an iframe in your own admin panel. Returns the URL string."""
|
|
518
|
+
r = _api("POST", A_EMBED, {"ttl": max(60, min(int(ttl or 3600), 86400))})
|
|
519
|
+
return r.get("url") if isinstance(r, dict) else None
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""python -m linearjeph.prophecy scan [dir] [--send] [--json] [--out file]
|
|
2
|
+
python -m linearjeph.prophecy import <events|customers> <file.csv|file.json>
|
|
3
|
+
Keys: LINEARJEPH_INGEST_KEY (scan --send), LINEARJEPH_SECRET_KEY (import). Endpoint: LINEARJEPH_ENDPOINT (default https://linearjeph.cc)."""
|
|
4
|
+
import csv
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
from . import ProphecyError, import_history, init, scan, send_manifest
|
|
12
|
+
|
|
13
|
+
USAGE = __doc__
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_rows(path, kind):
|
|
17
|
+
with io.open(path, "r", encoding="utf-8-sig", newline="") as f:
|
|
18
|
+
if path.lower().endswith(".csv"):
|
|
19
|
+
return [{k.strip(): v for k, v in row.items() if k and v not in (None, "")} for row in csv.DictReader(f)]
|
|
20
|
+
j = json.load(f)
|
|
21
|
+
return j if isinstance(j, list) else (j.get(kind) or j.get("rows") or [])
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(argv=None):
|
|
25
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
26
|
+
args = [a for a in argv if not a.startswith("--")]
|
|
27
|
+
flag = lambda n: "--" + n in argv # noqa: E731
|
|
28
|
+
if not args or args[0] not in ("scan", "import"):
|
|
29
|
+
sys.stderr.write(USAGE + "\n")
|
|
30
|
+
return 2
|
|
31
|
+
init()
|
|
32
|
+
if args[0] == "scan":
|
|
33
|
+
d = args[1] if len(args) > 1 else "."
|
|
34
|
+
t0 = time.time()
|
|
35
|
+
m = scan(d)
|
|
36
|
+
if "--out" in argv and argv.index("--out") + 1 < len(argv):
|
|
37
|
+
with open(argv[argv.index("--out") + 1], "w") as f:
|
|
38
|
+
json.dump(m, f, indent=2)
|
|
39
|
+
if flag("json"):
|
|
40
|
+
print(json.dumps(m, indent=2))
|
|
41
|
+
sys.stderr.write("scanned %s in %d ms: %d lines, %d routes, %d dependencies, %d features, %d integrations, "
|
|
42
|
+
"%d suspected secrets (counted, never sent)\n" % (
|
|
43
|
+
os.path.abspath(d), (time.time() - t0) * 1000, m["loc"], len(m["routes"]), len(m["dependencies"]),
|
|
44
|
+
len(m["features"]), len(m["integrations"]), m["security"]["secrets_suspected"]))
|
|
45
|
+
if flag("send"):
|
|
46
|
+
ok = send_manifest(m)
|
|
47
|
+
sys.stderr.write("manifest sent\n" if ok else "manifest NOT sent (check LINEARJEPH_INGEST_KEY and the endpoint)\n")
|
|
48
|
+
return 0 if ok else 1
|
|
49
|
+
return 0
|
|
50
|
+
if len(args) < 3 or args[1] not in ("events", "customers"):
|
|
51
|
+
sys.stderr.write(USAGE + "\n")
|
|
52
|
+
return 2
|
|
53
|
+
try:
|
|
54
|
+
rows = load_rows(args[2], args[1])
|
|
55
|
+
r = import_history(args[1], rows, on_progress=lambda p: sys.stderr.write(
|
|
56
|
+
"imported %(done)d/%(total)d (accepted %(accepted)d, rejected %(rejected)d)\n" % p))
|
|
57
|
+
except (ProphecyError, OSError, ValueError) as e:
|
|
58
|
+
sys.stderr.write("error: %s\n" % e)
|
|
59
|
+
return 1
|
|
60
|
+
sys.stderr.write("done: %d accepted, %d rejected\n" % (r["accepted"], r["rejected"]))
|
|
61
|
+
for e in r["errors"]:
|
|
62
|
+
sys.stderr.write(" %s\n" % json.dumps(e))
|
|
63
|
+
return 1 if r["rejected"] and not r["accepted"] else 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == "__main__":
|
|
67
|
+
sys.exit(main())
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Django middleware: MIDDLEWARE = [..., "linearjeph.prophecy.django.ProphecyMiddleware"]. Records the URL route template
|
|
2
|
+
(request.resolver_match.route), method, status and duration. Never bodies, headers or query strings."""
|
|
3
|
+
import asyncio
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
from .middleware import _record, normalize_path, route_template
|
|
7
|
+
|
|
8
|
+
try: # Django >= 4.2 provides the helper; older versions use the asyncio marker
|
|
9
|
+
from asgiref.sync import markcoroutinefunction as _mark
|
|
10
|
+
except ImportError: # pragma: no cover
|
|
11
|
+
def _mark(f):
|
|
12
|
+
f._is_coroutine = asyncio.coroutines._is_coroutine
|
|
13
|
+
return f
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ProphecyMiddleware(object):
|
|
17
|
+
sync_capable = True
|
|
18
|
+
async_capable = True
|
|
19
|
+
|
|
20
|
+
def __init__(self, get_response, ignore_paths=()):
|
|
21
|
+
self.get_response = get_response
|
|
22
|
+
self.ignore = tuple(ignore_paths or ())
|
|
23
|
+
self._async = asyncio.iscoroutinefunction(get_response)
|
|
24
|
+
if self._async:
|
|
25
|
+
_mark(self)
|
|
26
|
+
|
|
27
|
+
def __call__(self, request):
|
|
28
|
+
if self._async:
|
|
29
|
+
return self.__acall__(request)
|
|
30
|
+
t0 = time.perf_counter()
|
|
31
|
+
status = 500
|
|
32
|
+
try:
|
|
33
|
+
response = self.get_response(request)
|
|
34
|
+
status = getattr(response, "status_code", 500)
|
|
35
|
+
return response
|
|
36
|
+
finally:
|
|
37
|
+
self._done(request, status, t0)
|
|
38
|
+
|
|
39
|
+
async def __acall__(self, request):
|
|
40
|
+
t0 = time.perf_counter()
|
|
41
|
+
status = 500
|
|
42
|
+
try:
|
|
43
|
+
response = await self.get_response(request)
|
|
44
|
+
status = getattr(response, "status_code", 500)
|
|
45
|
+
return response
|
|
46
|
+
finally:
|
|
47
|
+
self._done(request, status, t0)
|
|
48
|
+
|
|
49
|
+
def _done(self, request, status, t0):
|
|
50
|
+
try:
|
|
51
|
+
path = getattr(request, "path", "/")
|
|
52
|
+
if self.ignore and path.startswith(self.ignore):
|
|
53
|
+
return
|
|
54
|
+
match = getattr(request, "resolver_match", None)
|
|
55
|
+
route = getattr(match, "route", None)
|
|
56
|
+
uid = None
|
|
57
|
+
user = getattr(request, "user", None)
|
|
58
|
+
if user is not None and getattr(user, "is_authenticated", False):
|
|
59
|
+
uid = getattr(user, "pk", None)
|
|
60
|
+
_record(route_template(route) if route else normalize_path(path), getattr(request, "method", "GET"), status,
|
|
61
|
+
(time.perf_counter() - t0) * 1000, uid)
|
|
62
|
+
except Exception: # noqa: BLE001 - never break a response
|
|
63
|
+
pass
|