TicketMetric-tool 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. ticketmetric_tool-0.1.2/PKG-INFO +304 -0
  2. ticketmetric_tool-0.1.2/README.md +278 -0
  3. ticketmetric_tool-0.1.2/pyproject.toml +50 -0
  4. ticketmetric_tool-0.1.2/setup.cfg +4 -0
  5. ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/PKG-INFO +304 -0
  6. ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/SOURCES.txt +36 -0
  7. ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/dependency_links.txt +1 -0
  8. ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/entry_points.txt +3 -0
  9. ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/requires.txt +13 -0
  10. ticketmetric_tool-0.1.2/src/TicketMetric_tool.egg-info/top_level.txt +1 -0
  11. ticketmetric_tool-0.1.2/src/tmcommon/__init__.py +27 -0
  12. ticketmetric_tool-0.1.2/src/tmcommon/config.py +78 -0
  13. ticketmetric_tool-0.1.2/src/tmcommon/context.py +73 -0
  14. ticketmetric_tool-0.1.2/src/tmcommon/exceptions.py +96 -0
  15. ticketmetric_tool-0.1.2/src/tmcommon/flask_ext.py +97 -0
  16. ticketmetric_tool-0.1.2/src/tmcommon/jobs/__init__.py +57 -0
  17. ticketmetric_tool-0.1.2/src/tmcommon/jobs/cli.py +104 -0
  18. ticketmetric_tool-0.1.2/src/tmcommon/jobs/job.py +239 -0
  19. ticketmetric_tool-0.1.2/src/tmcommon/jobs/lock.py +144 -0
  20. ticketmetric_tool-0.1.2/src/tmcommon/jobs/scheduler.py +139 -0
  21. ticketmetric_tool-0.1.2/src/tmcommon/logging.py +137 -0
  22. ticketmetric_tool-0.1.2/src/tmcommon/migrations/__init__.py +27 -0
  23. ticketmetric_tool-0.1.2/src/tmcommon/migrations/cli.py +78 -0
  24. ticketmetric_tool-0.1.2/src/tmcommon/migrations/runner.py +201 -0
  25. ticketmetric_tool-0.1.2/src/tmcommon/mongo/__init__.py +28 -0
  26. ticketmetric_tool-0.1.2/src/tmcommon/mongo/batch.py +132 -0
  27. ticketmetric_tool-0.1.2/src/tmcommon/mongo/checkpoint.py +80 -0
  28. ticketmetric_tool-0.1.2/src/tmcommon/mongo/connection.py +127 -0
  29. ticketmetric_tool-0.1.2/src/tmcommon/mongo/iterate.py +111 -0
  30. ticketmetric_tool-0.1.2/src/tmcommon/notify/__init__.py +3 -0
  31. ticketmetric_tool-0.1.2/src/tmcommon/notify/slack.py +172 -0
  32. ticketmetric_tool-0.1.2/src/tmcommon/retry.py +84 -0
  33. ticketmetric_tool-0.1.2/src/tmcommon/serialization.py +77 -0
  34. ticketmetric_tool-0.1.2/src/tmcommon/timing.py +80 -0
  35. ticketmetric_tool-0.1.2/tests/test_cli.py +121 -0
  36. ticketmetric_tool-0.1.2/tests/test_flask_ext.py +110 -0
  37. ticketmetric_tool-0.1.2/tests/test_jobs.py +168 -0
  38. ticketmetric_tool-0.1.2/tests/test_mongo_and_migrations.py +252 -0
@@ -0,0 +1,304 @@
1
+ Metadata-Version: 2.4
2
+ Name: TicketMetric_tool
3
+ Version: 0.1.2
4
+ Summary: Shared internal helpers: config, logging, Mongo, retries, migrations, jobs, notifications.
5
+ License-Expression: LicenseRef-Proprietary
6
+ Project-URL: Source, https://github.com/aryansingh3/common_internal_tool
7
+ Classifier: Programming Language :: Python :: 3.9
8
+ Classifier: Programming Language :: Python :: 3.10
9
+ Classifier: Programming Language :: Python :: 3.11
10
+ Classifier: Programming Language :: Python :: 3.12
11
+ Classifier: Programming Language :: Python :: 3.13
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Topic :: Software Development :: Libraries
14
+ Requires-Python: >=3.9
15
+ Description-Content-Type: text/markdown
16
+ Requires-Dist: pymongo>=4.6.0
17
+ Requires-Dist: python-dotenv>=1.0.0
18
+ Requires-Dist: requests>=2.31.0
19
+ Provides-Extra: flask
20
+ Requires-Dist: Flask>=2.3.0; extra == "flask"
21
+ Provides-Extra: logging
22
+ Requires-Dist: pretty-pie-log>=0.1.0; extra == "logging"
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=7.4; extra == "dev"
25
+ Requires-Dist: mongomock>=4.1; extra == "dev"
26
+
27
+ # TicketMetric_tool
28
+
29
+ Shared internal helpers for the API, the scrapers, the crons and one-off migrations.
30
+
31
+ > **Two names, on purpose.** The pip distribution is
32
+ > `TicketMetric_tool`; the import is the short `tmcommon`.
33
+ > Same convention as `pip install python-dateutil` / `import dateutil`, so call
34
+ > sites stay readable instead of carrying a 36 character prefix.
35
+ >
36
+ > ```bash
37
+ > pip install TicketMetric_tool
38
+ > ```
39
+ > ```python
40
+ > from tmcommon.jobs import Job
41
+ > ```
42
+ >
43
+ > To make them identical instead, rename `src/tmcommon/` and update the two
44
+ > `[project.scripts]` paths in `pyproject.toml`.
45
+
46
+ Design rule: **the library ships mechanism, the app keeps policy and schema.** Connection
47
+ handling, batching, retries and migration plumbing live here. Collection names, index
48
+ definitions and business queries stay in the app.
49
+
50
+ It is importable with no Flask installed, no Slack configured and no database reachable,
51
+ because migrations and scrapers need it too.
52
+
53
+ ## Install
54
+
55
+ ```bash
56
+ # from the repo
57
+ pip install -e /path/to/common_internal_tool # core
58
+ pip install -e ".[flask,logging]" # API services
59
+
60
+ # straight from GitHub
61
+ pip install "git+https://github.com/aryansingh3/common_internal_tool.git@main"
62
+ ```
63
+
64
+ Pin a release once there is one, rather than tracking `main`:
65
+
66
+ ```bash
67
+ pip install "git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.0"
68
+ ```
69
+
70
+ Build a wheel to hand around or host yourself:
71
+
72
+ ```bash
73
+ python -m build --wheel # dist/ticketmetric_tool-0.1.2-py3-none-any.whl
74
+ ```
75
+
76
+ ## Getting changes into the consuming repos
77
+
78
+ pip does not watch GitHub. Installing takes a snapshot, so a push to `main` changes
79
+ nothing in an environment that already installed the package. Pick the workflow that
80
+ matches where you are:
81
+
82
+ **Developing the library and an app together.** Clone once, install editable, and every
83
+ `git pull` is live with no reinstall. This is the only setup where changes really are
84
+ automatic.
85
+
86
+ ```bash
87
+ git clone https://github.com/aryansingh3/common_internal_tool.git
88
+ pip install -e ../common_internal_tool
89
+ ```
90
+
91
+ **Pulling the latest `main` into an environment.** pip caches aggressively, so an
92
+ `--upgrade` alone often appears to do nothing when the version string has not moved:
93
+
94
+ ```bash
95
+ pip install --upgrade --force-reinstall --no-cache-dir \
96
+ "git+https://github.com/aryansingh3/common_internal_tool.git@main"
97
+ ```
98
+
99
+ **Deployments.** Pin a tag, never `main`. A deploy that resolves `main` is not
100
+ reproducible, and the same Dockerfile will produce different images on different days.
101
+
102
+ ```
103
+ # requirements.txt
104
+ TicketMetric_tool @ git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.2
105
+ ```
106
+
107
+ Bumping that pin is then a reviewable one-line diff, which is what you want for a library
108
+ several services depend on.
109
+
110
+ ### Cutting a release
111
+
112
+ ```bash
113
+ # bump version in pyproject.toml, commit, then
114
+ git tag v0.2.0 && git push origin v0.2.0
115
+ ```
116
+
117
+ The release workflow verifies the tag matches `pyproject.toml`, runs the suite, builds the
118
+ wheel and sdist, and attaches them to a GitHub Release.
119
+
120
+ ### The private-repo gotcha
121
+
122
+ This repo is private, so anything without your local git credentials, CI, Docker builds,
123
+ production hosts, cannot clone it. A `pip install git+https://...` there fails with an
124
+ authentication error. The usual fixes:
125
+
126
+ - a deploy key on this repo, with the private key as a secret in the consumer, using the
127
+ `git+ssh://git@github.com/...` form
128
+ - a fine-grained PAT with read access, injected as
129
+ `git+https://${TOKEN}@github.com/...`
130
+ - or build the wheel in CI and push it to a private index
131
+
132
+ For Docker, mount the credential as a build secret rather than baking a token into a layer.
133
+
134
+ ## Python support
135
+
136
+ **3.9 through 3.13.** 3.9 is the floor because the Flask API runs on it, so nothing here
137
+ uses 3.10+ syntax: no PEP 604 `X | Y` unions, no `match` statements. CI compiles every
138
+ file on each version to enforce that, including modules the tests do not import.
139
+
140
+ ## Tests
141
+
142
+ ```bash
143
+ python3 run_tests.py
144
+ ```
145
+
146
+ No pytest needed, so the suite runs on an interpreter carrying only the core
147
+ dependencies. The files are valid pytest modules too. A test file whose optional
148
+ dependency is missing prints `SKIP` and passes, so no Flask is not a failure.
149
+
150
+ Verified locally on 3.9.6 and 3.11.15; 3.10, 3.12 and 3.13 are covered by the CI matrix.
151
+
152
+ `tests/fake_mongo.py` is a small in-memory `Collection` covering only the operations these
153
+ modules use, so lock and batch logic is testable without mongomock and without pointing a
154
+ test at a real cluster.
155
+
156
+ ## Environment
157
+
158
+ | variable | required | purpose |
159
+ |---|---|---|
160
+ | `SERVICE_NAME` | recommended | identifies the service in logs and alerts |
161
+ | `APP_ENV` | no | `dev`/`local`/`test`/`staging` suppress Slack; anything else is production |
162
+ | `LOG_DIR` | no | absolute log directory, defaults to `<cwd>/logs` |
163
+ | `LOG_LEVEL` | no | defaults to `DEBUG` |
164
+ | `MONGODB_URI` | when using mongo | default cluster |
165
+ | `MONGODB_URI_<ALIAS>` | no | extra clusters, e.g. `MONGODB_URI_STAGING` |
166
+ | `SLACK_WEBHOOK_URL` | when notifying | absent means notifications are skipped, not an error |
167
+
168
+ ## Modules
169
+
170
+ | module | what it gives you |
171
+ |---|---|
172
+ | `config` | typed env access that raises at point of use, not on import |
173
+ | `logging` | `get_logger(name)`, cached, absolute log dir, falls back to stdlib when `pretty_pie_log` is absent |
174
+ | `context` | correlation id and extra fields via ContextVar, so a cron traceback is traceable |
175
+ | `exceptions` | `BaseAPIException` and the HTTP subclasses, dependency-free |
176
+ | `serialization` | one JSON encoder for ObjectId and datetime, plus `to_jsonable` |
177
+ | `retry` | `@retry` with exponential backoff and jitter |
178
+ | `timing` | `timed()`, `@timeit`, and `Stopwatch` for multi-phase jobs |
179
+ | `mongo.connection` | lazy multi-cluster registry, `DatabaseRouter` for the live/historical split |
180
+ | `mongo.batch` | chunked `insert_many` / `bulk_write` that separates duplicate keys from real errors |
181
+ | `mongo.iterate` | `iter_by_id` keyset paging, no cursor timeouts, resumable |
182
+ | `mongo.checkpoint` | watermarks so incremental jobs resume instead of guessing a `--since` |
183
+ | `migrations` | versioned migrations with an applied record, a lock and dry-run |
184
+ | `jobs` | scheduled work: expiring locks, run history, heartbeats, overdue detection |
185
+ | `notify.slack` | error alerts, framework-agnostic, context supplied by the app |
186
+ | `flask_ext` | `init_app(app)` for correlation ids and error handlers |
187
+
188
+ ## Jobs and crons
189
+
190
+ A `Job` gets a lock, a run row, a status document with heartbeat, a correlation id, a
191
+ checkpoint, failure alerting and timing. Records use the field names already in
192
+ `vc_worker_runs` and `vc_worker_status`, so existing queries keep working.
193
+
194
+ ```python
195
+ from tmcommon.jobs import Job
196
+
197
+ class SectionRosterJob(Job):
198
+ name = 'tm_section_roster'
199
+ description = 'Fold new TM snapshots into the section roster'
200
+ lock_ttl_seconds = 1800
201
+
202
+ def run(self, ctx):
203
+ since = ctx.checkpoint.get()
204
+ processed = 0
205
+ for event_id in events_since(since):
206
+ update_roster(event_id)
207
+ processed += 1
208
+ if not ctx.heartbeat(processed=processed):
209
+ break # lease lost, another process took over
210
+ ctx.checkpoint.advance(newest_seen)
211
+ return {'events': processed}
212
+ ```
213
+
214
+ Run it in-process, replacing the hand-rolled worker threads:
215
+
216
+ ```python
217
+ from tmcommon.jobs import Scheduler
218
+
219
+ scheduler = Scheduler(db)
220
+ scheduler.add(SectionRosterJob(), interval_seconds=86400)
221
+ scheduler.add(CleanupJob(), interval_seconds=86400, initial_delay_seconds=300)
222
+ scheduler.start()
223
+ ```
224
+
225
+ Or as a one-shot under system cron or a Kubernetes CronJob, with the same recording:
226
+
227
+ ```bash
228
+ tm-job run --package jobs --db tickets --name tm_section_roster
229
+ tm-job status --db tickets
230
+ tm-job overdue --db tickets --expect tm_section_roster=86400,cleanup_worker=86400
231
+ ```
232
+
233
+ `overdue` exits non-zero when a job is late, so a monitor can alert on it. That is the
234
+ failure currently invisible: `cron.py` is a bare `while True` loop, and if that process
235
+ dies every schedule inside it stops with no signal.
236
+
237
+ **Why the lock matters here.** `app.py` starts the workers at module scope, in the `else`
238
+ branch gunicorn takes, so every web worker process starts its own copy. Four gunicorn
239
+ workers run four cleanup loops concurrently today. Under `Scheduler` all four still start
240
+ and exactly one does the work per tick; the rest record `skipped`.
241
+
242
+ ## Flask wiring
243
+
244
+ ```python
245
+ from tmcommon.flask_ext import init_app, flask_context_provider
246
+ from tmcommon.notify import register_context_provider
247
+
248
+ init_app(app)
249
+ register_context_provider(lambda: {**flask_context_provider(),
250
+ 'User': get_request_user().email})
251
+ ```
252
+
253
+ `BaseAPIException` becomes its `to_dict()` response and is logged at warning. Anything else
254
+ is treated as a bug: logged with a traceback, sent to Slack, returned as a 500. A 404 or 405
255
+ is neither.
256
+
257
+ ## Migrations
258
+
259
+ ```python
260
+ # migrations/m20261001_section_roster.py
261
+ from tmcommon.migrations import Migration
262
+
263
+ class SectionRoster(Migration):
264
+ version = '20261001_section_roster'
265
+ description = 'Build tm_section_roster from ticketmaster_detail_data'
266
+
267
+ def up(self, db, dry_run=False):
268
+ count = db.ticketmaster_detail_data.estimated_document_count()
269
+ if dry_run:
270
+ return {'would_scan': count}
271
+ ...
272
+ return {'events': 412, 'sections': 9304}
273
+ ```
274
+
275
+ ```bash
276
+ tm-migrate status --package migrations --db tickets
277
+ tm-migrate up --package migrations --db tickets # dry run
278
+ tm-migrate up --package migrations --db tickets --apply
279
+ ```
280
+
281
+ Writes need `--apply`. Deliberate, given these run against collections with millions of rows.
282
+
283
+ ## Deliberately not here
284
+
285
+ - **Collection accessors and index definitions.** Your `DatabaseConnection._create_indexes()`
286
+ is app schema. Declare it in the app and call it at startup.
287
+ - **`DatabaseManager` business queries.** `get_user_from_email`, `get_analytics` and the
288
+ venue lookups are domain logic, not shared mechanism.
289
+ - **Domain models** such as `APIUser` and `Membership`.
290
+
291
+ ## Roadmap
292
+
293
+ Not built yet, in rough priority order:
294
+
295
+ 1. `http` client: session with retry, proxy rotation and UA rotation. Both repos already carry
296
+ `proxies.json` and `user_agent.json` plus their own rotation code.
297
+ 2. `pipeline` base: extract/transform/load with stats and failure reporting, to replace the
298
+ per-spider boilerplate.
299
+ 3. `ratelimit`: token bucket for upstream politeness.
300
+ 4. `dates`: `est_to_utc`, `convert_utc_to_timezone`, currently duplicated in both repos.
301
+ 5. `mongo.upsert`: find-or-create keyed on an alternate identity, the gap behind the duplicate
302
+ events bug.
303
+ 6. `schema`: declarative model base with `to_dict` / `from_dict` and validation.
304
+ 7. Test helpers: `mongomock` fixtures and a fake clock.
@@ -0,0 +1,278 @@
1
+ # TicketMetric_tool
2
+
3
+ Shared internal helpers for the API, the scrapers, the crons and one-off migrations.
4
+
5
+ > **Two names, on purpose.** The pip distribution is
6
+ > `TicketMetric_tool`; the import is the short `tmcommon`.
7
+ > Same convention as `pip install python-dateutil` / `import dateutil`, so call
8
+ > sites stay readable instead of carrying a 36 character prefix.
9
+ >
10
+ > ```bash
11
+ > pip install TicketMetric_tool
12
+ > ```
13
+ > ```python
14
+ > from tmcommon.jobs import Job
15
+ > ```
16
+ >
17
+ > To make them identical instead, rename `src/tmcommon/` and update the two
18
+ > `[project.scripts]` paths in `pyproject.toml`.
19
+
20
+ Design rule: **the library ships mechanism, the app keeps policy and schema.** Connection
21
+ handling, batching, retries and migration plumbing live here. Collection names, index
22
+ definitions and business queries stay in the app.
23
+
24
+ It is importable with no Flask installed, no Slack configured and no database reachable,
25
+ because migrations and scrapers need it too.
26
+
27
+ ## Install
28
+
29
+ ```bash
30
+ # from the repo
31
+ pip install -e /path/to/common_internal_tool # core
32
+ pip install -e ".[flask,logging]" # API services
33
+
34
+ # straight from GitHub
35
+ pip install "git+https://github.com/aryansingh3/common_internal_tool.git@main"
36
+ ```
37
+
38
+ Pin a release once there is one, rather than tracking `main`:
39
+
40
+ ```bash
41
+ pip install "git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.0"
42
+ ```
43
+
44
+ Build a wheel to hand around or host yourself:
45
+
46
+ ```bash
47
+ python -m build --wheel # dist/ticketmetric_tool-0.1.2-py3-none-any.whl
48
+ ```
49
+
50
+ ## Getting changes into the consuming repos
51
+
52
+ pip does not watch GitHub. Installing takes a snapshot, so a push to `main` changes
53
+ nothing in an environment that already installed the package. Pick the workflow that
54
+ matches where you are:
55
+
56
+ **Developing the library and an app together.** Clone once, install editable, and every
57
+ `git pull` is live with no reinstall. This is the only setup where changes really are
58
+ automatic.
59
+
60
+ ```bash
61
+ git clone https://github.com/aryansingh3/common_internal_tool.git
62
+ pip install -e ../common_internal_tool
63
+ ```
64
+
65
+ **Pulling the latest `main` into an environment.** pip caches aggressively, so an
66
+ `--upgrade` alone often appears to do nothing when the version string has not moved:
67
+
68
+ ```bash
69
+ pip install --upgrade --force-reinstall --no-cache-dir \
70
+ "git+https://github.com/aryansingh3/common_internal_tool.git@main"
71
+ ```
72
+
73
+ **Deployments.** Pin a tag, never `main`. A deploy that resolves `main` is not
74
+ reproducible, and the same Dockerfile will produce different images on different days.
75
+
76
+ ```
77
+ # requirements.txt
78
+ TicketMetric_tool @ git+https://github.com/aryansingh3/common_internal_tool.git@v0.1.2
79
+ ```
80
+
81
+ Bumping that pin is then a reviewable one-line diff, which is what you want for a library
82
+ several services depend on.
83
+
84
+ ### Cutting a release
85
+
86
+ ```bash
87
+ # bump version in pyproject.toml, commit, then
88
+ git tag v0.2.0 && git push origin v0.2.0
89
+ ```
90
+
91
+ The release workflow verifies the tag matches `pyproject.toml`, runs the suite, builds the
92
+ wheel and sdist, and attaches them to a GitHub Release.
93
+
94
+ ### The private-repo gotcha
95
+
96
+ This repo is private, so anything without your local git credentials, CI, Docker builds,
97
+ production hosts, cannot clone it. A `pip install git+https://...` there fails with an
98
+ authentication error. The usual fixes:
99
+
100
+ - a deploy key on this repo, with the private key as a secret in the consumer, using the
101
+ `git+ssh://git@github.com/...` form
102
+ - a fine-grained PAT with read access, injected as
103
+ `git+https://${TOKEN}@github.com/...`
104
+ - or build the wheel in CI and push it to a private index
105
+
106
+ For Docker, mount the credential as a build secret rather than baking a token into a layer.
107
+
108
+ ## Python support
109
+
110
+ **3.9 through 3.13.** 3.9 is the floor because the Flask API runs on it, so nothing here
111
+ uses 3.10+ syntax: no PEP 604 `X | Y` unions, no `match` statements. CI compiles every
112
+ file on each version to enforce that, including modules the tests do not import.
113
+
114
+ ## Tests
115
+
116
+ ```bash
117
+ python3 run_tests.py
118
+ ```
119
+
120
+ No pytest needed, so the suite runs on an interpreter carrying only the core
121
+ dependencies. The files are valid pytest modules too. A test file whose optional
122
+ dependency is missing prints `SKIP` and passes, so no Flask is not a failure.
123
+
124
+ Verified locally on 3.9.6 and 3.11.15; 3.10, 3.12 and 3.13 are covered by the CI matrix.
125
+
126
+ `tests/fake_mongo.py` is a small in-memory `Collection` covering only the operations these
127
+ modules use, so lock and batch logic is testable without mongomock and without pointing a
128
+ test at a real cluster.
129
+
130
+ ## Environment
131
+
132
+ | variable | required | purpose |
133
+ |---|---|---|
134
+ | `SERVICE_NAME` | recommended | identifies the service in logs and alerts |
135
+ | `APP_ENV` | no | `dev`/`local`/`test`/`staging` suppress Slack; anything else is production |
136
+ | `LOG_DIR` | no | absolute log directory, defaults to `<cwd>/logs` |
137
+ | `LOG_LEVEL` | no | defaults to `DEBUG` |
138
+ | `MONGODB_URI` | when using mongo | default cluster |
139
+ | `MONGODB_URI_<ALIAS>` | no | extra clusters, e.g. `MONGODB_URI_STAGING` |
140
+ | `SLACK_WEBHOOK_URL` | when notifying | absent means notifications are skipped, not an error |
141
+
142
+ ## Modules
143
+
144
+ | module | what it gives you |
145
+ |---|---|
146
+ | `config` | typed env access that raises at point of use, not on import |
147
+ | `logging` | `get_logger(name)`, cached, absolute log dir, falls back to stdlib when `pretty_pie_log` is absent |
148
+ | `context` | correlation id and extra fields via ContextVar, so a cron traceback is traceable |
149
+ | `exceptions` | `BaseAPIException` and the HTTP subclasses, dependency-free |
150
+ | `serialization` | one JSON encoder for ObjectId and datetime, plus `to_jsonable` |
151
+ | `retry` | `@retry` with exponential backoff and jitter |
152
+ | `timing` | `timed()`, `@timeit`, and `Stopwatch` for multi-phase jobs |
153
+ | `mongo.connection` | lazy multi-cluster registry, `DatabaseRouter` for the live/historical split |
154
+ | `mongo.batch` | chunked `insert_many` / `bulk_write` that separates duplicate keys from real errors |
155
+ | `mongo.iterate` | `iter_by_id` keyset paging, no cursor timeouts, resumable |
156
+ | `mongo.checkpoint` | watermarks so incremental jobs resume instead of guessing a `--since` |
157
+ | `migrations` | versioned migrations with an applied record, a lock and dry-run |
158
+ | `jobs` | scheduled work: expiring locks, run history, heartbeats, overdue detection |
159
+ | `notify.slack` | error alerts, framework-agnostic, context supplied by the app |
160
+ | `flask_ext` | `init_app(app)` for correlation ids and error handlers |
161
+
162
+ ## Jobs and crons
163
+
164
+ A `Job` gets a lock, a run row, a status document with heartbeat, a correlation id, a
165
+ checkpoint, failure alerting and timing. Records use the field names already in
166
+ `vc_worker_runs` and `vc_worker_status`, so existing queries keep working.
167
+
168
+ ```python
169
+ from tmcommon.jobs import Job
170
+
171
+ class SectionRosterJob(Job):
172
+ name = 'tm_section_roster'
173
+ description = 'Fold new TM snapshots into the section roster'
174
+ lock_ttl_seconds = 1800
175
+
176
+ def run(self, ctx):
177
+ since = ctx.checkpoint.get()
178
+ processed = 0
179
+ for event_id in events_since(since):
180
+ update_roster(event_id)
181
+ processed += 1
182
+ if not ctx.heartbeat(processed=processed):
183
+ break # lease lost, another process took over
184
+ ctx.checkpoint.advance(newest_seen)
185
+ return {'events': processed}
186
+ ```
187
+
188
+ Run it in-process, replacing the hand-rolled worker threads:
189
+
190
+ ```python
191
+ from tmcommon.jobs import Scheduler
192
+
193
+ scheduler = Scheduler(db)
194
+ scheduler.add(SectionRosterJob(), interval_seconds=86400)
195
+ scheduler.add(CleanupJob(), interval_seconds=86400, initial_delay_seconds=300)
196
+ scheduler.start()
197
+ ```
198
+
199
+ Or as a one-shot under system cron or a Kubernetes CronJob, with the same recording:
200
+
201
+ ```bash
202
+ tm-job run --package jobs --db tickets --name tm_section_roster
203
+ tm-job status --db tickets
204
+ tm-job overdue --db tickets --expect tm_section_roster=86400,cleanup_worker=86400
205
+ ```
206
+
207
+ `overdue` exits non-zero when a job is late, so a monitor can alert on it. That is the
208
+ failure currently invisible: `cron.py` is a bare `while True` loop, and if that process
209
+ dies every schedule inside it stops with no signal.
210
+
211
+ **Why the lock matters here.** `app.py` starts the workers at module scope, in the `else`
212
+ branch gunicorn takes, so every web worker process starts its own copy. Four gunicorn
213
+ workers run four cleanup loops concurrently today. Under `Scheduler` all four still start
214
+ and exactly one does the work per tick; the rest record `skipped`.
215
+
216
+ ## Flask wiring
217
+
218
+ ```python
219
+ from tmcommon.flask_ext import init_app, flask_context_provider
220
+ from tmcommon.notify import register_context_provider
221
+
222
+ init_app(app)
223
+ register_context_provider(lambda: {**flask_context_provider(),
224
+ 'User': get_request_user().email})
225
+ ```
226
+
227
+ `BaseAPIException` becomes its `to_dict()` response and is logged at warning. Anything else
228
+ is treated as a bug: logged with a traceback, sent to Slack, returned as a 500. A 404 or 405
229
+ is neither.
230
+
231
+ ## Migrations
232
+
233
+ ```python
234
+ # migrations/m20261001_section_roster.py
235
+ from tmcommon.migrations import Migration
236
+
237
+ class SectionRoster(Migration):
238
+ version = '20261001_section_roster'
239
+ description = 'Build tm_section_roster from ticketmaster_detail_data'
240
+
241
+ def up(self, db, dry_run=False):
242
+ count = db.ticketmaster_detail_data.estimated_document_count()
243
+ if dry_run:
244
+ return {'would_scan': count}
245
+ ...
246
+ return {'events': 412, 'sections': 9304}
247
+ ```
248
+
249
+ ```bash
250
+ tm-migrate status --package migrations --db tickets
251
+ tm-migrate up --package migrations --db tickets # dry run
252
+ tm-migrate up --package migrations --db tickets --apply
253
+ ```
254
+
255
+ Writes need `--apply`. Deliberate, given these run against collections with millions of rows.
256
+
257
+ ## Deliberately not here
258
+
259
+ - **Collection accessors and index definitions.** Your `DatabaseConnection._create_indexes()`
260
+ is app schema. Declare it in the app and call it at startup.
261
+ - **`DatabaseManager` business queries.** `get_user_from_email`, `get_analytics` and the
262
+ venue lookups are domain logic, not shared mechanism.
263
+ - **Domain models** such as `APIUser` and `Membership`.
264
+
265
+ ## Roadmap
266
+
267
+ Not built yet, in rough priority order:
268
+
269
+ 1. `http` client: session with retry, proxy rotation and UA rotation. Both repos already carry
270
+ `proxies.json` and `user_agent.json` plus their own rotation code.
271
+ 2. `pipeline` base: extract/transform/load with stats and failure reporting, to replace the
272
+ per-spider boilerplate.
273
+ 3. `ratelimit`: token bucket for upstream politeness.
274
+ 4. `dates`: `est_to_utc`, `convert_utc_to_timezone`, currently duplicated in both repos.
275
+ 5. `mongo.upsert`: find-or-create keyed on an alternate identity, the gap behind the duplicate
276
+ events bug.
277
+ 6. `schema`: declarative model base with `to_dict` / `from_dict` and validation.
278
+ 7. Test helpers: `mongomock` fixtures and a fake clock.
@@ -0,0 +1,50 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ # Distribution name: what you `pip install`.
7
+ # The import name stays the short `tmcommon` (see README), the same way
8
+ # python-dateutil installs under one name and imports as `dateutil`.
9
+ name = "TicketMetric_tool"
10
+ version = "0.1.2"
11
+ description = "Shared internal helpers: config, logging, Mongo, retries, migrations, jobs, notifications."
12
+ # 3.9 is the floor because the Flask API runs on it. Nothing here uses 3.10+
13
+ # syntax (no PEP 604 unions, no match statements); CI enforces that.
14
+ requires-python = ">=3.9"
15
+ readme = "README.md"
16
+ license = "LicenseRef-Proprietary"
17
+ classifiers = [
18
+ "Programming Language :: Python :: 3.9",
19
+ "Programming Language :: Python :: 3.10",
20
+ "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12",
22
+ "Programming Language :: Python :: 3.13",
23
+ "Intended Audience :: Developers",
24
+ "Topic :: Software Development :: Libraries",
25
+ ]
26
+
27
+ dependencies = [
28
+ "pymongo>=4.6.0",
29
+ "python-dotenv>=1.0.0",
30
+ "requests>=2.31.0",
31
+ ]
32
+
33
+ [project.urls]
34
+ Source = "https://github.com/aryansingh3/common_internal_tool"
35
+
36
+ [project.optional-dependencies]
37
+ # Kept optional so a scraper or a migration script never has to install Flask.
38
+ flask = ["Flask>=2.3.0"]
39
+ logging = ["pretty-pie-log>=0.1.0"]
40
+ dev = ["pytest>=7.4", "mongomock>=4.1"]
41
+
42
+ [project.scripts]
43
+ tm-migrate = "tmcommon.migrations.cli:main"
44
+ tm-job = "tmcommon.jobs.cli:main"
45
+
46
+ [tool.setuptools.packages.find]
47
+ where = ["src"]
48
+
49
+ [tool.pytest.ini_options]
50
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+