TicketMetric-tool 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ticketmetric_tool-0.1.2/PKG-INFO +304 -0
- ticketmetric_tool-0.1.2/README.md +278 -0
- ticketmetric_tool-0.1.2/pyproject.toml +50 -0
- ticketmetric_tool-0.1.2/setup.cfg +4 -0
- ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/PKG-INFO +304 -0
- ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/SOURCES.txt +36 -0
- ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/dependency_links.txt +1 -0
- ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/entry_points.txt +3 -0
- ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/requires.txt +13 -0
- ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/top_level.txt +1 -0
- ticketmetric_tool-0.1.2/src/tmcommon/__init__.py +27 -0
- ticketmetric_tool-0.1.2/src/tmcommon/config.py +78 -0
- ticketmetric_tool-0.1.2/src/tmcommon/context.py +73 -0
- ticketmetric_tool-0.1.2/src/tmcommon/exceptions.py +96 -0
- ticketmetric_tool-0.1.2/src/tmcommon/flask_ext.py +97 -0
- ticketmetric_tool-0.1.2/src/tmcommon/jobs/__init__.py +57 -0
- ticketmetric_tool-0.1.2/src/tmcommon/jobs/cli.py +104 -0
- ticketmetric_tool-0.1.2/src/tmcommon/jobs/job.py +239 -0
- ticketmetric_tool-0.1.2/src/tmcommon/jobs/lock.py +144 -0
- ticketmetric_tool-0.1.2/src/tmcommon/jobs/scheduler.py +139 -0
- ticketmetric_tool-0.1.2/src/tmcommon/logging.py +137 -0
- ticketmetric_tool-0.1.2/src/tmcommon/migrations/__init__.py +27 -0
- ticketmetric_tool-0.1.2/src/tmcommon/migrations/cli.py +78 -0
- ticketmetric_tool-0.1.2/src/tmcommon/migrations/runner.py +201 -0
- ticketmetric_tool-0.1.2/src/tmcommon/mongo/__init__.py +28 -0
- ticketmetric_tool-0.1.2/src/tmcommon/mongo/batch.py +132 -0
- ticketmetric_tool-0.1.2/src/tmcommon/mongo/checkpoint.py +80 -0
- ticketmetric_tool-0.1.2/src/tmcommon/mongo/connection.py +127 -0
- ticketmetric_tool-0.1.2/src/tmcommon/mongo/iterate.py +111 -0
- ticketmetric_tool-0.1.2/src/tmcommon/notify/__init__.py +3 -0
- ticketmetric_tool-0.1.2/src/tmcommon/notify/slack.py +172 -0
- ticketmetric_tool-0.1.2/src/tmcommon/retry.py +84 -0
- ticketmetric_tool-0.1.2/src/tmcommon/serialization.py +77 -0
- ticketmetric_tool-0.1.2/src/tmcommon/timing.py +80 -0
- ticketmetric_tool-0.1.2/tests/test_cli.py +121 -0
- ticketmetric_tool-0.1.2/tests/test_flask_ext.py +110 -0
- ticketmetric_tool-0.1.2/tests/test_jobs.py +168 -0
- ticketmetric_tool-0.1.2/tests/test_mongo_and_migrations.py +252 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: TicketMetric_tool
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Shared internal helpers: config, logging, Mongo, retries, migrations, jobs, notifications.
|
|
5
|
+
License-Expression: LicenseRef-Proprietary
|
|
6
|
+
Project-URL: Source, https://github.com/aryansingh3/common_internal_tool
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Requires-Dist: pymongo>=4.6.0
|
|
17
|
+
Requires-Dist: python-dotenv>=1.0.0
|
|
18
|
+
Requires-Dist: requests>=2.31.0
|
|
19
|
+
Provides-Extra: flask
|
|
20
|
+
Requires-Dist: Flask>=2.3.0; extra == "flask"
|
|
21
|
+
Provides-Extra: logging
|
|
22
|
+
Requires-Dist: pretty-pie-log>=0.1.0; extra == "logging"
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=7.4; extra == "dev"
|
|
25
|
+
Requires-Dist: mongomock>=4.1; extra == "dev"
|
|
26
|
+
|
|
27
|
+
# TicketMetric_tool
|
|
28
|
+
|
|
29
|
+
Shared internal helpers for the API, the scrapers, the crons and one-off migrations.
|
|
30
|
+
|
|
31
|
+
> **Two names, on purpose.** The pip distribution is
|
|
32
|
+
> `TicketMetric_tool`; the import is the short `tmcommon`.
|
|
33
|
+
> Same convention as `pip install python-dateutil` / `import dateutil`, so call
|
|
34
|
+
> sites stay readable instead of carrying a 36 character prefix.
|
|
35
|
+
>
|
|
36
|
+
> ```bash
|
|
37
|
+
> pip install TicketMetric_tool
|
|
38
|
+
> ```
|
|
39
|
+
> ```python
|
|
40
|
+
> from tmcommon.jobs import Job
|
|
41
|
+
> ```
|
|
42
|
+
>
|
|
43
|
+
> To make them identical instead, rename `src/tmcommon/` and update the two
|
|
44
|
+
> `[project.scripts]` paths in `pyproject.toml`.
|
|
45
|
+
|
|
46
|
+
Design rule: **the library ships mechanism, the app keeps policy and schema.** Connection
|
|
47
|
+
handling, batching, retries and migration plumbing live here. Collection names, index
|
|
48
|
+
definitions and business queries stay in the app.
|
|
49
|
+
|
|
50
|
+
It is importable with no Flask installed, no Slack configured and no database reachable,
|
|
51
|
+
because migrations and scrapers need it too.
|
|
52
|
+
|
|
53
|
+
## Install
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
# from the repo
|
|
57
|
+
pip install -e /path/to/common_internal_tool # core
|
|
58
|
+
pip install -e ".[flask,logging]" # API services
|
|
59
|
+
|
|
60
|
+
# straight from GitHub
|
|
61
|
+
pip install "git+https://github.com/aryansingh3/common_internal_tool.git@main"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Pin a release once there is one, rather than tracking `main`:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install "git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.0"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Build a wheel to hand around or host yourself:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
python -m build --wheel # dist/ticketmetric_tool-0.1.2-py3-none-any.whl
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Getting changes into the consuming repos
|
|
77
|
+
|
|
78
|
+
pip does not watch GitHub. Installing takes a snapshot, so a push to `main` changes
|
|
79
|
+
nothing in an environment that already installed the package. Pick the workflow that
|
|
80
|
+
matches where you are:
|
|
81
|
+
|
|
82
|
+
**Developing the library and an app together.** Clone once, install editable, and every
|
|
83
|
+
`git pull` is live with no reinstall. This is the only setup where changes really are
|
|
84
|
+
automatic.
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
git clone https://github.com/aryansingh3/common_internal_tool.git
|
|
88
|
+
pip install -e ../common_internal_tool
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
**Pulling the latest `main` into an environment.** pip caches aggressively, so an
|
|
92
|
+
`--upgrade` alone often appears to do nothing when the version string has not moved:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
pip install --upgrade --force-reinstall --no-cache-dir \
|
|
96
|
+
"git+https://github.com/aryansingh3/common_internal_tool.git@main"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
**Deployments.** Pin a tag, never `main`. A deploy that resolves `main` is not
|
|
100
|
+
reproducible, and the same Dockerfile will produce different images on different days.
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
# requirements.txt
|
|
104
|
+
TicketMetric_tool @ git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.2
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Bumping that pin is then a reviewable one-line diff, which is what you want for a library
|
|
108
|
+
several services depend on.
|
|
109
|
+
|
|
110
|
+
### Cutting a release
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
# bump version in pyproject.toml, commit, then
|
|
114
|
+
git tag v0.2.0 && git push origin v0.2.0
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
The release workflow verifies the tag matches `pyproject.toml`, runs the suite, builds the
|
|
118
|
+
wheel and sdist, and attaches them to a GitHub Release.
|
|
119
|
+
|
|
120
|
+
### The private-repo gotcha
|
|
121
|
+
|
|
122
|
+
This repo is private, so anything without your local git credentials, CI, Docker builds,
|
|
123
|
+
production hosts, cannot clone it. A `pip install git+https://...` there fails with an
|
|
124
|
+
authentication error. The usual fixes:
|
|
125
|
+
|
|
126
|
+
- a deploy key on this repo, with the private key as a secret in the consumer, using the
|
|
127
|
+
`git+ssh://git@github.com/...` form
|
|
128
|
+
- a fine-grained PAT with read access, injected as
|
|
129
|
+
`git+https://${TOKEN}@github.com/...`
|
|
130
|
+
- or build the wheel in CI and push it to a private index
|
|
131
|
+
|
|
132
|
+
For Docker, mount the credential as a build secret rather than baking a token into a layer.
|
|
133
|
+
|
|
134
|
+
## Python support
|
|
135
|
+
|
|
136
|
+
**3.9 through 3.13.** 3.9 is the floor because the Flask API runs on it, so nothing here
|
|
137
|
+
uses 3.10+ syntax: no PEP 604 `X | Y` unions, no `match` statements. CI compiles every
|
|
138
|
+
file on each version to enforce that, including modules the tests do not import.
|
|
139
|
+
|
|
140
|
+
## Tests
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
python3 run_tests.py
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
No pytest needed, so the suite runs on an interpreter carrying only the core
|
|
147
|
+
dependencies. The files are valid pytest modules too. A test file whose optional
|
|
148
|
+
dependency is missing prints `SKIP` and passes, so no Flask is not a failure.
|
|
149
|
+
|
|
150
|
+
Verified locally on 3.9.6 and 3.11.15; 3.10, 3.12 and 3.13 are covered by the CI matrix.
|
|
151
|
+
|
|
152
|
+
`tests/fake_mongo.py` is a small in-memory `Collection` covering only the operations these
|
|
153
|
+
modules use, so lock and batch logic is testable without mongomock and without pointing a
|
|
154
|
+
test at a real cluster.
|
|
155
|
+
|
|
156
|
+
## Environment
|
|
157
|
+
|
|
158
|
+
| variable | required | purpose |
|
|
159
|
+
|---|---|---|
|
|
160
|
+
| `SERVICE_NAME` | recommended | identifies the service in logs and alerts |
|
|
161
|
+
| `APP_ENV` | no | `dev`/`local`/`test`/`staging` suppress Slack; anything else is production |
|
|
162
|
+
| `LOG_DIR` | no | absolute log directory, defaults to `<cwd>/logs` |
|
|
163
|
+
| `LOG_LEVEL` | no | defaults to `DEBUG` |
|
|
164
|
+
| `MONGODB_URI` | when using mongo | default cluster |
|
|
165
|
+
| `MONGODB_URI_<ALIAS>` | no | extra clusters, e.g. `MONGODB_URI_STAGING` |
|
|
166
|
+
| `SLACK_WEBHOOK_URL` | when notifying | absent means notifications are skipped, not an error |
|
|
167
|
+
|
|
168
|
+
## Modules
|
|
169
|
+
|
|
170
|
+
| module | what it gives you |
|
|
171
|
+
|---|---|
|
|
172
|
+
| `config` | typed env access that raises at point of use, not on import |
|
|
173
|
+
| `logging` | `get_logger(name)`, cached, absolute log dir, falls back to stdlib when `pretty_pie_log` is absent |
|
|
174
|
+
| `context` | correlation id and extra fields via ContextVar, so a cron traceback is traceable |
|
|
175
|
+
| `exceptions` | `BaseAPIException` and the HTTP subclasses, dependency-free |
|
|
176
|
+
| `serialization` | one JSON encoder for ObjectId and datetime, plus `to_jsonable` |
|
|
177
|
+
| `retry` | `@retry` with exponential backoff and jitter |
|
|
178
|
+
| `timing` | `timed()`, `@timeit`, and `Stopwatch` for multi-phase jobs |
|
|
179
|
+
| `mongo.connection` | lazy multi-cluster registry, `DatabaseRouter` for the live/historical split |
|
|
180
|
+
| `mongo.batch` | chunked `insert_many` / `bulk_write` that separates duplicate keys from real errors |
|
|
181
|
+
| `mongo.iterate` | `iter_by_id` keyset paging, no cursor timeouts, resumable |
|
|
182
|
+
| `mongo.checkpoint` | watermarks so incremental jobs resume instead of guessing a `--since` |
|
|
183
|
+
| `migrations` | versioned migrations with an applied record, a lock and dry-run |
|
|
184
|
+
| `jobs` | scheduled work: expiring locks, run history, heartbeats, overdue detection |
|
|
185
|
+
| `notify.slack` | error alerts, framework-agnostic, context supplied by the app |
|
|
186
|
+
| `flask_ext` | `init_app(app)` for correlation ids and error handlers |
|
|
187
|
+
|
|
188
|
+
## Jobs and crons
|
|
189
|
+
|
|
190
|
+
A `Job` gets a lock, a run row, a status document with heartbeat, a correlation id, a
|
|
191
|
+
checkpoint, failure alerting and timing. Records use the field names already in
|
|
192
|
+
`vc_worker_runs` and `vc_worker_status`, so existing queries keep working.
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
from tmcommon.jobs import Job
|
|
196
|
+
|
|
197
|
+
class SectionRosterJob(Job):
|
|
198
|
+
name = 'tm_section_roster'
|
|
199
|
+
description = 'Fold new TM snapshots into the section roster'
|
|
200
|
+
lock_ttl_seconds = 1800
|
|
201
|
+
|
|
202
|
+
def run(self, ctx):
|
|
203
|
+
since = ctx.checkpoint.get()
|
|
204
|
+
processed = 0
|
|
205
|
+
for event_id in events_since(since):
|
|
206
|
+
update_roster(event_id)
|
|
207
|
+
processed += 1
|
|
208
|
+
if not ctx.heartbeat(processed=processed):
|
|
209
|
+
break # lease lost, another process took over
|
|
210
|
+
ctx.checkpoint.advance(newest_seen)
|
|
211
|
+
return {'events': processed}
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
Run it in-process, replacing the hand-rolled worker threads:
|
|
215
|
+
|
|
216
|
+
```python
|
|
217
|
+
from tmcommon.jobs import Scheduler
|
|
218
|
+
|
|
219
|
+
scheduler = Scheduler(db)
|
|
220
|
+
scheduler.add(SectionRosterJob(), interval_seconds=86400)
|
|
221
|
+
scheduler.add(CleanupJob(), interval_seconds=86400, initial_delay_seconds=300)
|
|
222
|
+
scheduler.start()
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Or as a one-shot under system cron or a Kubernetes CronJob, with the same recording:
|
|
226
|
+
|
|
227
|
+
```bash
|
|
228
|
+
tm-job run --package jobs --db tickets --name tm_section_roster
|
|
229
|
+
tm-job status --db tickets
|
|
230
|
+
tm-job overdue --db tickets --expect tm_section_roster=86400,cleanup_worker=86400
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
`overdue` exits non-zero when a job is late, so a monitor can alert on it. That is the
|
|
234
|
+
failure currently invisible: `cron.py` is a bare `while True` loop, and if that process
|
|
235
|
+
dies every schedule inside it stops with no signal.
|
|
236
|
+
|
|
237
|
+
**Why the lock matters here.** `app.py` starts the workers at module scope, in the `else`
|
|
238
|
+
branch gunicorn takes, so every web worker process starts its own copy. Four gunicorn
|
|
239
|
+
workers run four cleanup loops concurrently today. Under `Scheduler` all four still start
|
|
240
|
+
and exactly one does the work per tick; the rest record `skipped`.
|
|
241
|
+
|
|
242
|
+
## Flask wiring
|
|
243
|
+
|
|
244
|
+
```python
|
|
245
|
+
from tmcommon.flask_ext import init_app, flask_context_provider
|
|
246
|
+
from tmcommon.notify import register_context_provider
|
|
247
|
+
|
|
248
|
+
init_app(app)
|
|
249
|
+
register_context_provider(lambda: {**flask_context_provider(),
|
|
250
|
+
'User': get_request_user().email})
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
`BaseAPIException` becomes its `to_dict()` response and is logged at warning. Anything else
|
|
254
|
+
is treated as a bug: logged with a traceback, sent to Slack, returned as a 500. A 404 or 405
|
|
255
|
+
is neither.
|
|
256
|
+
|
|
257
|
+
## Migrations
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
# migrations/m20261001_section_roster.py
|
|
261
|
+
from tmcommon.migrations import Migration
|
|
262
|
+
|
|
263
|
+
class SectionRoster(Migration):
|
|
264
|
+
version = '20261001_section_roster'
|
|
265
|
+
description = 'Build tm_section_roster from ticketmaster_detail_data'
|
|
266
|
+
|
|
267
|
+
def up(self, db, dry_run=False):
|
|
268
|
+
count = db.ticketmaster_detail_data.estimated_document_count()
|
|
269
|
+
if dry_run:
|
|
270
|
+
return {'would_scan': count}
|
|
271
|
+
...
|
|
272
|
+
return {'events': 412, 'sections': 9304}
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
```bash
|
|
276
|
+
tm-migrate status --package migrations --db tickets
|
|
277
|
+
tm-migrate up --package migrations --db tickets # dry run
|
|
278
|
+
tm-migrate up --package migrations --db tickets --apply
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Writes need `--apply`. Deliberate, given these run against collections with millions of rows.
|
|
282
|
+
|
|
283
|
+
## Deliberately not here
|
|
284
|
+
|
|
285
|
+
- **Collection accessors and index definitions.** Your `DatabaseConnection._create_indexes()`
|
|
286
|
+
is app schema. Declare it in the app and call it at startup.
|
|
287
|
+
- **`DatabaseManager` business queries.** `get_user_from_email`, `get_analytics` and the
|
|
288
|
+
venue lookups are domain logic, not shared mechanism.
|
|
289
|
+
- **Domain models** such as `APIUser` and `Membership`.
|
|
290
|
+
|
|
291
|
+
## Roadmap
|
|
292
|
+
|
|
293
|
+
Not built yet, in rough priority order:
|
|
294
|
+
|
|
295
|
+
1. `http` client: session with retry, proxy rotation and UA rotation. Both repos already carry
|
|
296
|
+
`proxies.json` and `user_agent.json` plus their own rotation code.
|
|
297
|
+
2. `pipeline` base: extract/transform/load with stats and failure reporting, to replace the
|
|
298
|
+
per-spider boilerplate.
|
|
299
|
+
3. `ratelimit`: token bucket for upstream politeness.
|
|
300
|
+
4. `dates`: `est_to_utc`, `convert_utc_to_timezone`, currently duplicated in both repos.
|
|
301
|
+
5. `mongo.upsert`: find-or-create keyed on an alternate identity, the gap behind the duplicate
|
|
302
|
+
events bug.
|
|
303
|
+
6. `schema`: declarative model base with `to_dict` / `from_dict` and validation.
|
|
304
|
+
7. Test helpers: `mongomock` fixtures and a fake clock.
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
# TicketMetric_tool
|
|
2
|
+
|
|
3
|
+
Shared internal helpers for the API, the scrapers, the crons and one-off migrations.
|
|
4
|
+
|
|
5
|
+
> **Two names, on purpose.** The pip distribution is
|
|
6
|
+
> `TicketMetric_tool`; the import is the short `tmcommon`.
|
|
7
|
+
> Same convention as `pip install python-dateutil` / `import dateutil`, so call
|
|
8
|
+
> sites stay readable instead of carrying a 36 character prefix.
|
|
9
|
+
>
|
|
10
|
+
> ```bash
|
|
11
|
+
> pip install TicketMetric_tool
|
|
12
|
+
> ```
|
|
13
|
+
> ```python
|
|
14
|
+
> from tmcommon.jobs import Job
|
|
15
|
+
> ```
|
|
16
|
+
>
|
|
17
|
+
> To make them identical instead, rename `src/tmcommon/` and update the two
|
|
18
|
+
> `[project.scripts]` paths in `pyproject.toml`.
|
|
19
|
+
|
|
20
|
+
Design rule: **the library ships mechanism, the app keeps policy and schema.** Connection
|
|
21
|
+
handling, batching, retries and migration plumbing live here. Collection names, index
|
|
22
|
+
definitions and business queries stay in the app.
|
|
23
|
+
|
|
24
|
+
It is importable with no Flask installed, no Slack configured and no database reachable,
|
|
25
|
+
because migrations and scrapers need it too.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
# from the repo
|
|
31
|
+
pip install -e /path/to/common_internal_tool # core
|
|
32
|
+
pip install -e ".[flask,logging]" # API services
|
|
33
|
+
|
|
34
|
+
# straight from GitHub
|
|
35
|
+
pip install "git+https://github.com/aryansingh3/common_internal_tool.git@main"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Pin a release once there is one, rather than tracking `main`:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.0"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Build a wheel to hand around or host yourself:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
python -m build --wheel # dist/ticketmetric_tool-0.1.2-py3-none-any.whl
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Getting changes into the consuming repos
|
|
51
|
+
|
|
52
|
+
pip does not watch GitHub. Installing takes a snapshot, so a push to `main` changes
|
|
53
|
+
nothing in an environment that already installed the package. Pick the workflow that
|
|
54
|
+
matches where you are:
|
|
55
|
+
|
|
56
|
+
**Developing the library and an app together.** Clone once, install editable, and every
|
|
57
|
+
`git pull` is live with no reinstall. This is the only setup where changes really are
|
|
58
|
+
automatic.
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
git clone https://github.com/aryansingh3/common_internal_tool.git
|
|
62
|
+
pip install -e ../common_internal_tool
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
**Pulling the latest `main` into an environment.** pip caches aggressively, so an
|
|
66
|
+
`--upgrade` alone often appears to do nothing when the version string has not moved:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
pip install --upgrade --force-reinstall --no-cache-dir \
|
|
70
|
+
"git+https://github.com/aryansingh3/common_internal_tool.git@main"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
**Deployments.** Pin a tag, never `main`. A deploy that resolves `main` is not
|
|
74
|
+
reproducible, and the same Dockerfile will produce different images on different days.
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
# requirements.txt
|
|
78
|
+
TicketMetric_tool @ git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.2
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Bumping that pin is then a reviewable one-line diff, which is what you want for a library
|
|
82
|
+
several services depend on.
|
|
83
|
+
|
|
84
|
+
### Cutting a release
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
# bump version in pyproject.toml, commit, then
|
|
88
|
+
git tag v0.2.0 && git push origin v0.2.0
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
The release workflow verifies the tag matches `pyproject.toml`, runs the suite, builds the
|
|
92
|
+
wheel and sdist, and attaches them to a GitHub Release.
|
|
93
|
+
|
|
94
|
+
### The private-repo gotcha
|
|
95
|
+
|
|
96
|
+
This repo is private, so anything without your local git credentials, CI, Docker builds,
|
|
97
|
+
production hosts, cannot clone it. A `pip install git+https://...` there fails with an
|
|
98
|
+
authentication error. The usual fixes:
|
|
99
|
+
|
|
100
|
+
- a deploy key on this repo, with the private key as a secret in the consumer, using the
|
|
101
|
+
`git+ssh://git@github.com/...` form
|
|
102
|
+
- a fine-grained PAT with read access, injected as
|
|
103
|
+
`git+https://${TOKEN}@github.com/...`
|
|
104
|
+
- or build the wheel in CI and push it to a private index
|
|
105
|
+
|
|
106
|
+
For Docker, mount the credential as a build secret rather than baking a token into a layer.
|
|
107
|
+
|
|
108
|
+
## Python support
|
|
109
|
+
|
|
110
|
+
**3.9 through 3.13.** 3.9 is the floor because the Flask API runs on it, so nothing here
|
|
111
|
+
uses 3.10+ syntax: no PEP 604 `X | Y` unions, no `match` statements. CI compiles every
|
|
112
|
+
file on each version to enforce that, including modules the tests do not import.
|
|
113
|
+
|
|
114
|
+
## Tests
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
python3 run_tests.py
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
No pytest needed, so the suite runs on an interpreter carrying only the core
|
|
121
|
+
dependencies. The files are valid pytest modules too. A test file whose optional
|
|
122
|
+
dependency is missing prints `SKIP` and passes, so no Flask is not a failure.
|
|
123
|
+
|
|
124
|
+
Verified locally on 3.9.6 and 3.11.15; 3.10, 3.12 and 3.13 are covered by the CI matrix.
|
|
125
|
+
|
|
126
|
+
`tests/fake_mongo.py` is a small in-memory `Collection` covering only the operations these
|
|
127
|
+
modules use, so lock and batch logic is testable without mongomock and without pointing a
|
|
128
|
+
test at a real cluster.
|
|
129
|
+
|
|
130
|
+
## Environment
|
|
131
|
+
|
|
132
|
+
| variable | required | purpose |
|
|
133
|
+
|---|---|---|
|
|
134
|
+
| `SERVICE_NAME` | recommended | identifies the service in logs and alerts |
|
|
135
|
+
| `APP_ENV` | no | `dev`/`local`/`test`/`staging` suppress Slack; anything else is production |
|
|
136
|
+
| `LOG_DIR` | no | absolute log directory, defaults to `<cwd>/logs` |
|
|
137
|
+
| `LOG_LEVEL` | no | defaults to `DEBUG` |
|
|
138
|
+
| `MONGODB_URI` | when using mongo | default cluster |
|
|
139
|
+
| `MONGODB_URI_<ALIAS>` | no | extra clusters, e.g. `MONGODB_URI_STAGING` |
|
|
140
|
+
| `SLACK_WEBHOOK_URL` | when notifying | absent means notifications are skipped, not an error |
|
|
141
|
+
|
|
142
|
+
## Modules
|
|
143
|
+
|
|
144
|
+
| module | what it gives you |
|
|
145
|
+
|---|---|
|
|
146
|
+
| `config` | typed env access that raises at point of use, not on import |
|
|
147
|
+
| `logging` | `get_logger(name)`, cached, absolute log dir, falls back to stdlib when `pretty_pie_log` is absent |
|
|
148
|
+
| `context` | correlation id and extra fields via ContextVar, so a cron traceback is traceable |
|
|
149
|
+
| `exceptions` | `BaseAPIException` and the HTTP subclasses, dependency-free |
|
|
150
|
+
| `serialization` | one JSON encoder for ObjectId and datetime, plus `to_jsonable` |
|
|
151
|
+
| `retry` | `@retry` with exponential backoff and jitter |
|
|
152
|
+
| `timing` | `timed()`, `@timeit`, and `Stopwatch` for multi-phase jobs |
|
|
153
|
+
| `mongo.connection` | lazy multi-cluster registry, `DatabaseRouter` for the live/historical split |
|
|
154
|
+
| `mongo.batch` | chunked `insert_many` / `bulk_write` that separates duplicate keys from real errors |
|
|
155
|
+
| `mongo.iterate` | `iter_by_id` keyset paging, no cursor timeouts, resumable |
|
|
156
|
+
| `mongo.checkpoint` | watermarks so incremental jobs resume instead of guessing a `--since` |
|
|
157
|
+
| `migrations` | versioned migrations with an applied record, a lock and dry-run |
|
|
158
|
+
| `jobs` | scheduled work: expiring locks, run history, heartbeats, overdue detection |
|
|
159
|
+
| `notify.slack` | error alerts, framework-agnostic, context supplied by the app |
|
|
160
|
+
| `flask_ext` | `init_app(app)` for correlation ids and error handlers |
|
|
161
|
+
|
|
162
|
+
## Jobs and crons
|
|
163
|
+
|
|
164
|
+
A `Job` gets a lock, a run row, a status document with heartbeat, a correlation id, a
|
|
165
|
+
checkpoint, failure alerting and timing. Records use the field names already in
|
|
166
|
+
`vc_worker_runs` and `vc_worker_status`, so existing queries keep working.
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from tmcommon.jobs import Job
|
|
170
|
+
|
|
171
|
+
class SectionRosterJob(Job):
|
|
172
|
+
name = 'tm_section_roster'
|
|
173
|
+
description = 'Fold new TM snapshots into the section roster'
|
|
174
|
+
lock_ttl_seconds = 1800
|
|
175
|
+
|
|
176
|
+
def run(self, ctx):
|
|
177
|
+
since = ctx.checkpoint.get()
|
|
178
|
+
processed = 0
|
|
179
|
+
for event_id in events_since(since):
|
|
180
|
+
update_roster(event_id)
|
|
181
|
+
processed += 1
|
|
182
|
+
if not ctx.heartbeat(processed=processed):
|
|
183
|
+
break # lease lost, another process took over
|
|
184
|
+
ctx.checkpoint.advance(newest_seen)
|
|
185
|
+
return {'events': processed}
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Run it in-process, replacing the hand-rolled worker threads:
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
from tmcommon.jobs import Scheduler
|
|
192
|
+
|
|
193
|
+
scheduler = Scheduler(db)
|
|
194
|
+
scheduler.add(SectionRosterJob(), interval_seconds=86400)
|
|
195
|
+
scheduler.add(CleanupJob(), interval_seconds=86400, initial_delay_seconds=300)
|
|
196
|
+
scheduler.start()
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Or as a one-shot under system cron or a Kubernetes CronJob, with the same recording:
|
|
200
|
+
|
|
201
|
+
```bash
|
|
202
|
+
tm-job run --package jobs --db tickets --name tm_section_roster
|
|
203
|
+
tm-job status --db tickets
|
|
204
|
+
tm-job overdue --db tickets --expect tm_section_roster=86400,cleanup_worker=86400
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
`overdue` exits non-zero when a job is late, so a monitor can alert on it. That is the
|
|
208
|
+
failure currently invisible: `cron.py` is a bare `while True` loop, and if that process
|
|
209
|
+
dies every schedule inside it stops with no signal.
|
|
210
|
+
|
|
211
|
+
**Why the lock matters here.** `app.py` starts the workers at module scope, in the `else`
|
|
212
|
+
branch gunicorn takes, so every web worker process starts its own copy. Four gunicorn
|
|
213
|
+
workers run four cleanup loops concurrently today. Under `Scheduler` all four still start
|
|
214
|
+
and exactly one does the work per tick; the rest record `skipped`.
|
|
215
|
+
|
|
216
|
+
## Flask wiring
|
|
217
|
+
|
|
218
|
+
```python
|
|
219
|
+
from tmcommon.flask_ext import init_app, flask_context_provider
|
|
220
|
+
from tmcommon.notify import register_context_provider
|
|
221
|
+
|
|
222
|
+
init_app(app)
|
|
223
|
+
register_context_provider(lambda: {**flask_context_provider(),
|
|
224
|
+
'User': get_request_user().email})
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
`BaseAPIException` becomes its `to_dict()` response and is logged at warning. Anything else
|
|
228
|
+
is treated as a bug: logged with a traceback, sent to Slack, returned as a 500. A 404 or 405
|
|
229
|
+
is neither.
|
|
230
|
+
|
|
231
|
+
## Migrations
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
# migrations/m20261001_section_roster.py
|
|
235
|
+
from tmcommon.migrations import Migration
|
|
236
|
+
|
|
237
|
+
class SectionRoster(Migration):
|
|
238
|
+
version = '20261001_section_roster'
|
|
239
|
+
description = 'Build tm_section_roster from ticketmaster_detail_data'
|
|
240
|
+
|
|
241
|
+
def up(self, db, dry_run=False):
|
|
242
|
+
count = db.ticketmaster_detail_data.estimated_document_count()
|
|
243
|
+
if dry_run:
|
|
244
|
+
return {'would_scan': count}
|
|
245
|
+
...
|
|
246
|
+
return {'events': 412, 'sections': 9304}
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
```bash
|
|
250
|
+
tm-migrate status --package migrations --db tickets
|
|
251
|
+
tm-migrate up --package migrations --db tickets # dry run
|
|
252
|
+
tm-migrate up --package migrations --db tickets --apply
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Writes need `--apply`. Deliberate, given these run against collections with millions of rows.
|
|
256
|
+
|
|
257
|
+
## Deliberately not here
|
|
258
|
+
|
|
259
|
+
- **Collection accessors and index definitions.** Your `DatabaseConnection._create_indexes()`
|
|
260
|
+
is app schema. Declare it in the app and call it at startup.
|
|
261
|
+
- **`DatabaseManager` business queries.** `get_user_from_email`, `get_analytics` and the
|
|
262
|
+
venue lookups are domain logic, not shared mechanism.
|
|
263
|
+
- **Domain models** such as `APIUser` and `Membership`.
|
|
264
|
+
|
|
265
|
+
## Roadmap
|
|
266
|
+
|
|
267
|
+
Not built yet, in rough priority order:
|
|
268
|
+
|
|
269
|
+
1. `http` client: session with retry, proxy rotation and UA rotation. Both repos already carry
|
|
270
|
+
`proxies.json` and `user_agent.json` plus their own rotation code.
|
|
271
|
+
2. `pipeline` base: extract/transform/load with stats and failure reporting, to replace the
|
|
272
|
+
per-spider boilerplate.
|
|
273
|
+
3. `ratelimit`: token bucket for upstream politeness.
|
|
274
|
+
4. `dates`: `est_to_utc`, `convert_utc_to_timezone`, currently duplicated in both repos.
|
|
275
|
+
5. `mongo.upsert`: find-or-create keyed on an alternate identity, the gap behind the duplicate
|
|
276
|
+
events bug.
|
|
277
|
+
6. `schema`: declarative model base with `to_dict` / `from_dict` and validation.
|
|
278
|
+
7. Test helpers: `mongomock` fixtures and a fake clock.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
# Distribution name: what you `pip install`.
|
|
7
|
+
# The import name stays the short `tmcommon` (see README), the same way
|
|
8
|
+
# python-dateutil installs under one name and imports as `dateutil`.
|
|
9
|
+
name = "TicketMetric_tool"
|
|
10
|
+
version = "0.1.2"
|
|
11
|
+
description = "Shared internal helpers: config, logging, Mongo, retries, migrations, jobs, notifications."
|
|
12
|
+
# 3.9 is the floor because the Flask API runs on it. Nothing here uses 3.10+
|
|
13
|
+
# syntax (no PEP 604 unions, no match statements); CI enforces that.
|
|
14
|
+
requires-python = ">=3.9"
|
|
15
|
+
readme = "README.md"
|
|
16
|
+
license = "LicenseRef-Proprietary"
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Programming Language :: Python :: 3.9",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Intended Audience :: Developers",
|
|
24
|
+
"Topic :: Software Development :: Libraries",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
dependencies = [
|
|
28
|
+
"pymongo>=4.6.0",
|
|
29
|
+
"python-dotenv>=1.0.0",
|
|
30
|
+
"requests>=2.31.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Source = "https://github.com/aryansingh3/common_internal_tool"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
# Kept optional so a scraper or a migration script never has to install Flask.
|
|
38
|
+
flask = ["Flask>=2.3.0"]
|
|
39
|
+
logging = ["pretty-pie-log>=0.1.0"]
|
|
40
|
+
dev = ["pytest>=7.4", "mongomock>=4.1"]
|
|
41
|
+
|
|
42
|
+
[project.scripts]
|
|
43
|
+
tm-migrate = "tmcommon.migrations.cli:main"
|
|
44
|
+
tm-job = "tmcommon.jobs.cli:main"
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.packages.find]
|
|
47
|
+
where = ["src"]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|