tgdata 0.0.9__tar.gz → 0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tgdata-0.0.9 → tgdata-0.1}/PKG-INFO +297 -122
- {tgdata-0.0.9 → tgdata-0.1}/README.md +295 -120
- {tgdata-0.0.9 → tgdata-0.1}/setup.py +5 -3
- tgdata-0.1/tgdata/__init__.py +184 -0
- tgdata-0.1/tgdata/account_operation.py +137 -0
- tgdata-0.1/tgdata/account_pool.py +691 -0
- tgdata-0.1/tgdata/backfill.py +498 -0
- tgdata-0.1/tgdata/backfill_engine.py +747 -0
- tgdata-0.1/tgdata/backfill_state.py +316 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/batch_engine.py +58 -24
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/budget_client.py +4 -4
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/connection_engine.py +79 -14
- tgdata-0.1/tgdata/group_operations.py +423 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/health.py +123 -29
- tgdata-0.1/tgdata/history_window.py +77 -0
- tgdata-0.1/tgdata/join_budget.py +311 -0
- tgdata-0.1/tgdata/join_client.py +108 -0
- tgdata-0.1/tgdata/login.py +524 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/message_batch.py +11 -0
- tgdata-0.1/tgdata/owned_health.py +239 -0
- tgdata-0.1/tgdata/pool_lifecycle.py +137 -0
- tgdata-0.1/tgdata/pool_observation.py +82 -0
- tgdata-0.1/tgdata/pool_result.py +138 -0
- tgdata-0.1/tgdata/pool_source.py +206 -0
- tgdata-0.1/tgdata/pool_state.py +236 -0
- tgdata-0.1/tgdata/progress_facade.py +140 -0
- tgdata-0.1/tgdata/smoke_tests/pool_storage_cases.py +419 -0
- tgdata-0.1/tgdata/smoke_tests/test_22_daily_continuation.py +758 -0
- tgdata-0.1/tgdata/smoke_tests/test_23_fixed_windows.py +489 -0
- tgdata-0.1/tgdata/smoke_tests/test_24_backfill_state.py +536 -0
- tgdata-0.1/tgdata/smoke_tests/test_25_backfill_delivery.py +836 -0
- tgdata-0.1/tgdata/smoke_tests/test_26_backfill_completion.py +436 -0
- tgdata-0.1/tgdata/smoke_tests/test_27_backfill_pacing.py +682 -0
- tgdata-0.1/tgdata/smoke_tests/test_28_backfill_controls.py +599 -0
- tgdata-0.1/tgdata/smoke_tests/test_29_backfill_public.py +487 -0
- tgdata-0.1/tgdata/smoke_tests/test_30_backfill_integration.py +403 -0
- tgdata-0.1/tgdata/smoke_tests/test_31_account_pool.py +1040 -0
- tgdata-0.1/tgdata/smoke_tests/test_32_account_operation.py +621 -0
- tgdata-0.1/tgdata/smoke_tests/test_33_owned_health.py +1045 -0
- tgdata-0.1/tgdata/smoke_tests/test_34_group_access.py +865 -0
- tgdata-0.1/tgdata/smoke_tests/test_35_join_budget.py +595 -0
- tgdata-0.1/tgdata/smoke_tests/test_36_group_join.py +865 -0
- tgdata-0.1/tgdata/smoke_tests/test_37_worker.py +683 -0
- tgdata-0.1/tgdata/smoke_tests/test_38_worker_process.py +644 -0
- tgdata-0.1/tgdata/smoke_tests/test_39_pool_lifecycle.py +490 -0
- tgdata-0.1/tgdata/smoke_tests/test_40_pool_outcomes.py +763 -0
- tgdata-0.1/tgdata/smoke_tests/test_41_stepwise_login.py +702 -0
- tgdata-0.1/tgdata/sync_engine.py +188 -0
- tgdata-0.1/tgdata/sync_store.py +364 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/tgdata.py +127 -66
- tgdata-0.1/tgdata/worker.py +264 -0
- tgdata-0.1/tgdata/worker_engine.py +280 -0
- tgdata-0.1/tgdata/worker_protocol.py +306 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata.egg-info/PKG-INFO +297 -122
- {tgdata-0.0.9 → tgdata-0.1}/tgdata.egg-info/SOURCES.txt +41 -1
- tgdata-0.1/tgdata.egg-info/entry_points.txt +2 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata.egg-info/requires.txt +1 -1
- tgdata-0.0.9/tgdata/__init__.py +0 -108
- tgdata-0.0.9/tgdata/group_operations.py +0 -328
- tgdata-0.0.9/tgdata/join_budget.py +0 -213
- tgdata-0.0.9/tgdata/join_client.py +0 -70
- tgdata-0.0.9/tgdata/smoke_tests/test_20_group_operations.py +0 -692
- {tgdata-0.0.9 → tgdata-0.1}/setup.cfg +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/batch_files.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/discovery_engine.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/message_engine.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/models.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/progress.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/read_budget.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/session_store.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/__init__.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/my_test.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_00_connection.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_01_list_group_chats.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_02_get_message_count.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_03_get_all_messages.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_04_get_all_messages_in_batches.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_05_with_progress_feature.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_06_polling_simple.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_07_on_new_message_simple.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_07_real_time_event_handler.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_08_batch_callback.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_08_polling.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_10_advanced_features.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_11_discover_groups.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_12_proxy.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_13_device_identity.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_14_flood_threshold.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_15_login_checks.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_16_health_events.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_17_session_store.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_18_read_budget.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_19_message_batches.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_debug_min_id.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_polling_debug.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/smoke_tests/test_username_support.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata/utils.py +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata.egg-info/dependency_links.txt +0 -0
- {tgdata-0.0.9 → tgdata-0.1}/tgdata.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tgdata
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.1
|
|
4
4
|
Summary: A production-grade Python library for extracting and processing Telegram group and channel messages
|
|
5
5
|
Author: enes kuzucu
|
|
6
6
|
Classifier: Development Status :: 3 - Alpha
|
|
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Requires-Python: >=3.7
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
|
-
Requires-Dist: Telethon<2.0,>=1.45
|
|
17
|
+
Requires-Dist: Telethon<2.0,>=1.45
|
|
18
18
|
Requires-Dist: pandas>=1.0
|
|
19
19
|
Provides-Extra: proxy
|
|
20
20
|
Requires-Dist: python-socks[asyncio]>=2.0; extra == "proxy"
|
|
@@ -59,6 +59,10 @@ A production-grade Python library for extracting and processing Telegram group a
|
|
|
59
59
|
- 🩺 **Account Health Events**: Every wait, logout, ban, restriction or lost group is reported as a plain-data event — through one callback, a log line and a per-account summary — and "ok" when it ends
|
|
60
60
|
- **Per-account read budgets**: An optional shared SQLite ledger limits message pulls over a rolling 24 hours, with persistent warm-up steps and recoverable partial results
|
|
61
61
|
- **Versioned message batches**: Read bounded raw snapshots with canonical JSON, replayable batch IDs, safe continuation cursors and optional content-addressed media
|
|
62
|
+
- **Durable daily continuation**: Keep per-group progress and pending batches in a pluggable store, replay after restart, and advance only after explicit acknowledgment
|
|
63
|
+
- **Reading worker**: Run a serial process that accepts explicit-account JSON jobs for batches, group lookup, access checks and joining, with correlated health replies
|
|
64
|
+
|
|
65
|
+
- **Account routing and failover**: An optional pool chooses among distinct existing accounts, enforces their budgets and preserves daily/backfill delivery; automatic joining is excluded
|
|
62
66
|
|
|
63
67
|
## Installation
|
|
64
68
|
|
|
@@ -66,12 +70,23 @@ A production-grade Python library for extracting and processing Telegram group a
|
|
|
66
70
|
pip install tgdata
|
|
67
71
|
```
|
|
68
72
|
|
|
73
|
+
## Reading worker
|
|
69
74
|
|
|
70
|
-
|
|
75
|
+
Run `python -m tgdata.worker --config worker.json` to accept serial JSON-line
|
|
76
|
+
jobs for an explicitly selected account: read a message batch, look up a group,
|
|
77
|
+
check access or join. The worker reuses existing sessions, proxy/device settings
|
|
78
|
+
and provisioned budgets, verifies the account for each job, and returns correlated
|
|
79
|
+
results and health events. It never requests a login code or retries a lost job.
|
|
71
80
|
|
|
72
|
-
|
|
81
|
+
See [the worker protocol and setup](docs/worker.md) for Telethon 1.45.0, inventory,
|
|
82
|
+
reply/exit semantics and session-store bootstrap. Try
|
|
83
|
+
`python examples/reading_worker.py --demo` for an offline process demonstration.
|
|
84
|
+
Scheduling, result storage and continuation acknowledgments stay with the caller.
|
|
85
|
+
|
|
86
|
+
## Authentication
|
|
73
87
|
|
|
74
|
-
|
|
88
|
+
Use terminal login for a local first run, or the explicit `LoginFlow` API for
|
|
89
|
+
an application that collects the phone, code and optional password itself.
|
|
75
90
|
|
|
76
91
|
1. get telegram development credentials in telegram API Development Tools from [https://my.telegram.org/apps](https://my.telegram.org/apps)
|
|
77
92
|
|
|
@@ -118,6 +133,36 @@ prompting.
|
|
|
118
133
|
Retrying cannot log it back in — and, unlike before, it no longer sends the
|
|
119
134
|
owner a fresh login code on every retry.
|
|
120
135
|
|
|
136
|
+
### Step-by-step login without console prompts
|
|
137
|
+
|
|
138
|
+
`LoginFlow` lets your application supply each input in a separate call:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
from tgdata import LoginFlow
|
|
142
|
+
|
|
143
|
+
flow = LoginFlow("config.ini")
|
|
144
|
+
status = await flow.start(phone) # Deliberately requests one code.
|
|
145
|
+
status = await flow.submit_code(code)
|
|
146
|
+
if status.step == "password":
|
|
147
|
+
status = await flow.submit_password(password)
|
|
148
|
+
if status.step == "complete" and not status.cleanup_pending:
|
|
149
|
+
# session_name must equal the configured session_file store key.
|
|
150
|
+
store.save(session_name, flow.session_data)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Inspect `status` after every call: waits, expired codes and unsupported Telegram
|
|
154
|
+
interactions need application handling. The flow uses the configured proxy and
|
|
155
|
+
device settings and never loads or replaces an existing session automatically.
|
|
156
|
+
The application saves the finished credential explicitly.
|
|
157
|
+
|
|
158
|
+
You can keep the flow in memory or store `flow.export_state()` privately and
|
|
159
|
+
restore it with `LoginFlow("config.ini", state=saved_state)`. Use only the latest
|
|
160
|
+
state with one owner at a time. The state is secret, not a browser token; restoring
|
|
161
|
+
it does not make an interrupted request safe to repeat. See the
|
|
162
|
+
[login guide](docs/stepwise_login.md) for errors, cancellation, cleanup and
|
|
163
|
+
session-store integration. Qualified offline against Telethon 1.45.0; live code
|
|
164
|
+
delivery and password acceptance remain unqualified.
|
|
165
|
+
|
|
121
166
|
### Keeping sessions out of files
|
|
122
167
|
|
|
123
168
|
Pass a store once to keep each account's session in your application's storage:
|
|
@@ -391,131 +436,84 @@ The budget suite exercises real Telethon request/iterator code with scripted
|
|
|
391
436
|
replies on 1.45.0. It does not test live Telegram response bounds or
|
|
392
437
|
whether any chosen limit prevents account restrictions.
|
|
393
438
|
|
|
394
|
-
|
|
439
|
+
### Joining with an account allowance
|
|
395
440
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
441
|
+
`JoinBudget` provides persistent per-account policy and status for admitted join
|
|
442
|
+
attempts in a rolling 24-hour window. Provision explicitly with
|
|
443
|
+
`JoinBudget("allowances.sqlite3", create=True)`, then reopen without `create=True`
|
|
444
|
+
in later jobs. `configure(account_id, daily_limit)` retains spent allowance;
|
|
445
|
+
zero pauses admission. `status(account_id).to_dict()` returns a local snapshot.
|
|
401
446
|
|
|
402
|
-
|
|
403
|
-
|
|
447
|
+
Pass that ledger to `TgData(config_path, join_budget=budget)` and call
|
|
448
|
+
`await tg.join_group(target, account_id=expected_numeric_id)`. A temporary client
|
|
449
|
+
verifies the account, resolves the target, and claims one attempt immediately before
|
|
450
|
+
each new SDK join/import send. Known membership or a known payment requirement
|
|
451
|
+
returns without a mutation or charge. Failed or interrupted admitted attempts are
|
|
452
|
+
not refunded.
|
|
404
453
|
|
|
405
|
-
|
|
406
|
-
|
|
454
|
+
The portable result distinguishes `joined`, `already_joined`, `requested`,
|
|
455
|
+
`payment_required` and `interaction_required`. Its group metadata is from preflight;
|
|
456
|
+
joining does not prove history access. Payments and bot interactions are left to the
|
|
457
|
+
caller. The join guard applies to this operation; raw clients remain outside it.
|
|
458
|
+
See [joining and outcomes](docs/group_joining.md) and
|
|
459
|
+
[the allowance contract](docs/join_budget.md) for configuration and failure behavior.
|
|
407
460
|
|
|
408
|
-
|
|
409
|
-
async with tg.connection_engine.ephemeral_client() as client:
|
|
410
|
-
me = await client.get_me()
|
|
411
|
-
budget.configure(me.id, daily_limit=3) # Illustrative policy, not a safe rate.
|
|
461
|
+
## Account routing and failover
|
|
412
462
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
463
|
+
Use `AccountPool` when tgdata should choose among distinct, already logged-in accounts
|
|
464
|
+
for a group read. Supply account configurations, existing budget policies, canonical
|
|
465
|
+
groups and persistent pool state. It supports raw batches and existing daily/backfill
|
|
466
|
+
operations. Plain `TgData` connection pooling still means sessions of one account.
|
|
416
467
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
print(error.status.to_dict(), error.retry_after)
|
|
468
|
+
```python
|
|
469
|
+
from tgdata import (
|
|
470
|
+
AccountPool, PoolAccount, PoolGroup, PoolReadCapture,
|
|
471
|
+
ReadBudget, SQLiteAccountPoolStore,
|
|
472
|
+
)
|
|
423
473
|
|
|
424
|
-
|
|
474
|
+
pool = AccountPool(
|
|
475
|
+
[PoolAccount(111111, "account-a.ini"), PoolAccount(222222, "account-b.ini")],
|
|
476
|
+
[PoolGroup(-1001234567890, "@my_existing_group")],
|
|
477
|
+
read_budget=ReadBudget("existing-read-budgets.sqlite3"),
|
|
478
|
+
state_store=SQLiteAccountPoolStore("existing-pool.sqlite3", create=False),
|
|
479
|
+
)
|
|
480
|
+
capture = PoolReadCapture() # Keep this reference outside the awaited read.
|
|
481
|
+
try:
|
|
482
|
+
result = await pool.read(-1001234567890, capture=capture, after_id=500, limit=100)
|
|
483
|
+
print(result.account_id, result.batch.batch_id)
|
|
484
|
+
finally:
|
|
485
|
+
snapshot = capture.snapshot() # Available complete records also survive failure/cancellation.
|
|
486
|
+
await pool.close()
|
|
425
487
|
```
|
|
426
488
|
|
|
427
|
-
`
|
|
428
|
-
|
|
429
|
-
and
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
read access. Access issues one logical history request with limit1 when possible,
|
|
456
|
-
including for nonmembers of public groups. Telethon may retry that request;
|
|
457
|
-
each actual send is bounded and metered by an optional ReadBudget. An empty
|
|
458
|
-
successful reply is still readable.
|
|
459
|
-
`GroupAccess.readable` is `True`, `False` or `None`; `reason` is a safe denial name
|
|
460
|
-
or `NO_PEER`. Authentication, transport, read-quota and local validation/storage
|
|
461
|
-
failures raise their actual error instead of becoming a denied result. An optional
|
|
462
|
-
`ReadBudget` meters this history request independently of the join allowance.
|
|
463
|
-
|
|
464
|
-
Joining requires a configured `JoinBudget` and account policy, even if preflight
|
|
465
|
-
finds existing membership. That observation returns `already_joined` without a
|
|
466
|
-
mutation or charge. `requested` means approval was requested; `interaction_required`
|
|
467
|
-
includes `bot_id` and `query_id`, without inventing a webview URL. Paid previews
|
|
468
|
-
return `payment_required` without sending a join. tgdata does not pay, open a
|
|
469
|
-
webview or wait for approval. Only `joined` and `already_joined` have `member=True`;
|
|
470
|
-
other join outcomes have `member=None`. None of these replaces a separate read check.
|
|
471
|
-
|
|
472
|
-
After a join acknowledgment, available returned details are cached locally. Missing
|
|
473
|
-
details or failed optional cache writes do not turn that acknowledgment into failed
|
|
474
|
-
joining; a cache failure logs its error type only. No extra lookup is required after
|
|
475
|
-
the acknowledgment. An unrecognized reply raises `GroupResponseError` and retains
|
|
476
|
-
the attempt charge.
|
|
477
|
-
|
|
478
|
-
The join allowance has these rules:
|
|
479
|
-
|
|
480
|
-
- **Unit:** one actual admitted join request. Every SDK retry costs another
|
|
481
|
-
attempt. Error replies (including already-participant or approval-requested),
|
|
482
|
-
transport loss, cancellation and crashes retain the charge. No uncertainty
|
|
483
|
-
refund is made. Preflight metadata and already-observed membership cost zero.
|
|
484
|
-
- **Storage:** all processes/clients sharing an allowance must use the same
|
|
485
|
-
SQLite file on the same host. Policies are keyed by fresh authenticated account
|
|
486
|
-
ID, never a session name or cached identity. The parent directory must exist;
|
|
487
|
-
`:memory:` is rejected. Only IDs, policy limits and admission times are stored.
|
|
488
|
-
ReadBudget can share the file; its tables and units remain separate.
|
|
489
|
-
- **Policy:** `configure(account_id, daily_limit)` requires explicit nonnegative
|
|
490
|
-
integer limits, preserves existing usage, and provides no default rate. Zero
|
|
491
|
-
stops new attempts. Lowering a limit does not erase prior attempts. The example
|
|
492
|
-
limit is not a claim about avoiding Telegram restrictions.
|
|
493
|
-
- **Time:** each attempt expires 24 hours after admission, with no midnight reset.
|
|
494
|
-
Backward clock movement cannot restore spent capacity; forward jumps advance
|
|
495
|
-
expiry, so keep the host clock accurate. SQLite claims are atomic and no lock
|
|
496
|
-
is held while awaiting Telegram. Synchronous operations are normally short;
|
|
497
|
-
contention can block the event loop for up to the five-second busy timeout.
|
|
498
|
-
- **Status:** local `budget.status(account_id)` or ephemeral
|
|
499
|
-
`await tg.get_join_budget()` returns `JoinBudgetStatus`. Without a budget the
|
|
500
|
-
latter returns `None` without connecting. Status reports `limit`, `used`,
|
|
501
|
-
`remaining`, observation/next-availability times and `retry_after` for one attempt.
|
|
502
|
-
Availability assumes no new claims/policy changes; a zero cap has no timed retry.
|
|
503
|
-
- **Failures and boundary:** missing policy, exhausted allowance and storage
|
|
504
|
-
failure stop before mutation (`JoinBudgetConfigError`, `JoinBudgetExceeded`,
|
|
505
|
-
`JoinBudgetStorageError`). Configured factory clients also guard raw join calls;
|
|
506
|
-
join-bearing batches and unknown wrappers raise `UnsupportedJoinRequest`.
|
|
507
|
-
Separately constructed clients, other ledger files, direct private transport
|
|
508
|
-
use and delayed approvals are outside this local admission guarantee.
|
|
509
|
-
|
|
510
|
-
Health events use the ephemeral call's fresh identity and a token-free invite
|
|
511
|
-
label. Metadata and join results cannot clear a prior group read-access denial;
|
|
512
|
-
a successful access read can. Local failures do not inherit unrelated Telegram
|
|
513
|
-
errors from the caller. New operation requests raise flood waits immediately;
|
|
514
|
-
the existing connection-authorization check keeps its usual wait behavior.
|
|
515
|
-
|
|
516
|
-
`test_20_group_operations` verifies SDK dispatch, results, lifecycle, health and
|
|
517
|
-
real SQLite concurrency offline with sockets blocked. It does not establish live
|
|
518
|
-
Telegram acceptance or a safe joining rate.
|
|
489
|
+
Pool `read`, `get_message_batch`, `sync_group` and `prepare_backfill` require a new
|
|
490
|
+
`capture=` for each call, including stored replay. Its synchronous `snapshot()` returns
|
|
491
|
+
immutable state and available data independently of exceptions or asyncio wrappers.
|
|
492
|
+
While the call is running, its batch contains only complete records with an
|
|
493
|
+
`interrupted` stop reason. Final success preserves the actual batch; errors and
|
|
494
|
+
cancellation preserve the complete interrupted prefix. A snapshot is not an
|
|
495
|
+
acknowledgment receipt. Ordinary `TgData` calls keep their signatures.
|
|
496
|
+
|
|
497
|
+
For first provisioning, create a new pool store and explicitly call `initialize()`;
|
|
498
|
+
do not recreate missing known state on restart. Completed partial batches stop failover
|
|
499
|
+
so their data can be delivered first. Unknown attempts require explicit recovery.
|
|
500
|
+
No eligible account raises structured `PoolUnavailable`, not empty history.
|
|
501
|
+
|
|
502
|
+
The adapter requires Telethon **1.45.0**, one owning process and exclusive sessions.
|
|
503
|
+
It never joins or prompts. Results describe the selected account's visible history.
|
|
504
|
+
Cancellation waits for the call's owned source/storage work to settle. Closing the
|
|
505
|
+
pool joins its own operation and retires clients without waiting for unrelated
|
|
506
|
+
application work after that operation ends. A daily/backfill publication can still
|
|
507
|
+
be running under its caller; its capture becomes final when that public call ends.
|
|
508
|
+
Provider-owned interruption is a local storage error, not caller cancellation.
|
|
509
|
+
Completed data remains available through the retained capture without implying a
|
|
510
|
+
durable receipt. Captures are kept in memory; existing progress stores govern replay.
|
|
511
|
+
Failed client retirement blocks reuse until an explicit recheck or close succeeds.
|
|
512
|
+
Use synchronous `pool.drain_health_events()` for bounded passive health observations;
|
|
513
|
+
AccountPool does not accept application health callbacks.
|
|
514
|
+
Offline tests pass; two-account live qualification remains pending. See
|
|
515
|
+
[Account pools](docs/account_pool.md) for selection, recovery and delivery contracts.
|
|
516
|
+
Run the offline example with `python examples/account_pool.py --demo`.
|
|
519
517
|
|
|
520
518
|
## Quick Start
|
|
521
519
|
|
|
@@ -544,6 +542,29 @@ Outputs:
|
|
|
544
542
|
Python Devs 10012312313
|
|
545
543
|
```
|
|
546
544
|
|
|
545
|
+
### Look up a group and check access
|
|
546
|
+
|
|
547
|
+
Use an expected numeric Telegram account ID for a short-lived, read-only observation:
|
|
548
|
+
|
|
549
|
+
```python
|
|
550
|
+
async with TgData("config.ini", interactive_login=False) as tg:
|
|
551
|
+
access = await tg.check_group_access("@example_group", account_id=123456789)
|
|
552
|
+
print(access.status) # readable, denied, or unprobed
|
|
553
|
+
if access.lookup is not None:
|
|
554
|
+
print(access.group.title, access.member)
|
|
555
|
+
```
|
|
556
|
+
|
|
557
|
+
`check_group_access` includes lookup details and probes at most one message of history
|
|
558
|
+
through the existing read budget. `lookup_group(target, account_id=...)` returns only
|
|
559
|
+
metadata and membership hints, without reading history. Both verify the actual account,
|
|
560
|
+
close their temporary client, and never log in or join. Operational failures still raise;
|
|
561
|
+
`unprobed` means there was no usable peer, and `denied` means a classified Telegram refusal.
|
|
562
|
+
|
|
563
|
+
Handles, supported Telegram links and deterministic numeric references are accepted.
|
|
564
|
+
Numeric channel IDs need a cached access hash. Readability is distinct from membership
|
|
565
|
+
and does not promise future access. See [the group-operations contract](docs/group_operations.md)
|
|
566
|
+
for input rules, result fields, errors, budget costs and the known SDK cache limitation.
|
|
567
|
+
|
|
547
568
|
### Group discovery — finding groups you are not in
|
|
548
569
|
|
|
549
570
|
`list_groups` shows what the account already follows. To find *new* rooms there are three
|
|
@@ -786,6 +807,108 @@ refuses cleanup may leave a private temporary file. The exact schema, limits,
|
|
|
786
807
|
interruption rules
|
|
787
808
|
and delivery protocol are in [Message batch v1](docs/message_batch_v1.md).
|
|
788
809
|
|
|
810
|
+
### Daily continuation across restarts
|
|
811
|
+
|
|
812
|
+
Use a progress store when tgdata should remember unfinished delivery and the last
|
|
813
|
+
accepted message for each group:
|
|
814
|
+
|
|
815
|
+
```python
|
|
816
|
+
from tgdata import TgData, SQLiteSyncStore
|
|
817
|
+
|
|
818
|
+
tg = TgData("config.ini", interactive_login=False,
|
|
819
|
+
sync_store=SQLiteSyncStore("daily-progress.sqlite3"))
|
|
820
|
+
chat_id = -1001234567890 # canonical MessageBatch.chat_id
|
|
821
|
+
|
|
822
|
+
# Initial setup: your existing archive's last accepted ID, or explicit 0.
|
|
823
|
+
await tg.initialize_sync(chat_id, after_id=48213)
|
|
824
|
+
try:
|
|
825
|
+
batch = await tg.sync_group(chat_id, limit=200)
|
|
826
|
+
if batch is not None:
|
|
827
|
+
await destination.accept(batch) # your durable, duplicate-safe destination
|
|
828
|
+
await tg.acknowledge_sync(chat_id, batch.batch_id)
|
|
829
|
+
finally:
|
|
830
|
+
await tg.close()
|
|
831
|
+
```
|
|
832
|
+
|
|
833
|
+
The next run replays a pending batch without contacting Telegram, or reads after
|
|
834
|
+
the acknowledged position. Preparing a batch never advances progress. Matching
|
|
835
|
+
repeated initialization does not reset it. `destination.accept` is your application
|
|
836
|
+
function; acknowledge only after its effects and duplicate markers are committed.
|
|
837
|
+
|
|
838
|
+
This first delivery supports one reader per group. Scheduling remains with the
|
|
839
|
+
caller; backfill is separate and edits/deletions are not reconciled. Optional media,
|
|
840
|
+
partial budget failures, custom storage and exact acknowledgment rules are covered
|
|
841
|
+
in [Daily continuation](docs/daily_continuation.md). Run the local example without
|
|
842
|
+
Telegram using `python examples/daily_continuation.py --demo`.
|
|
843
|
+
|
|
844
|
+
### Fixed historical windows
|
|
845
|
+
|
|
846
|
+
Give a historical collection its own progress store, then freeze its window at
|
|
847
|
+
first enrollment:
|
|
848
|
+
|
|
849
|
+
```python
|
|
850
|
+
from tgdata import TgData, SQLiteSyncStore
|
|
851
|
+
|
|
852
|
+
tg = TgData("config.ini", sync_store=SQLiteSyncStore("historical-progress.sqlite3"))
|
|
853
|
+
try:
|
|
854
|
+
await tg.initialize_sync(chat_id, after_id=0, last_days=30)
|
|
855
|
+
batch = await tg.sync_group(chat_id, limit=200)
|
|
856
|
+
if batch is not None:
|
|
857
|
+
await destination.accept(batch)
|
|
858
|
+
await tg.acknowledge_sync(chat_id, batch.batch_id)
|
|
859
|
+
finally:
|
|
860
|
+
await tg.close()
|
|
861
|
+
```
|
|
862
|
+
|
|
863
|
+
Repeated matching initialization reuses the saved dates. You can instead enroll
|
|
864
|
+
with timezone-aware `start_date` and `end_date`; the start is included and end
|
|
865
|
+
excluded. Daily and historical collections use separate stores, so their bookmarks
|
|
866
|
+
stay independent. See [Fixed historical windows](docs/fixed_windows.md) for date
|
|
867
|
+
limits, restart rules and direct batch reads. Scheduling and pacing remain with
|
|
868
|
+
the caller.
|
|
869
|
+
|
|
870
|
+
### Durable backfill runs
|
|
871
|
+
|
|
872
|
+
For historical reads with pause/cancel, restart recovery, pacing and explicit delivery
|
|
873
|
+
acknowledgments, configure a separate backfill progress store:
|
|
874
|
+
|
|
875
|
+
```python
|
|
876
|
+
from uuid import uuid4
|
|
877
|
+
from tgdata import TgData, SQLiteSyncStore, BackfillStartRequest, BackfillPrepareContext
|
|
878
|
+
|
|
879
|
+
tg = TgData("config.ini", interactive_login=False,
|
|
880
|
+
backfill_store=SQLiteSyncStore("backfill-progress.sqlite3"))
|
|
881
|
+
request = BackfillStartRequest(
|
|
882
|
+
collection_id="archive-history", chat_id=chat_id, run_id=uuid4().hex,
|
|
883
|
+
destination_id="my-archive", pause_seconds=240, expected_predecessor=None,
|
|
884
|
+
last_days=30, batch_size=200,
|
|
885
|
+
)
|
|
886
|
+
# Durably retain request.to_dict() in your application before submitting it.
|
|
887
|
+
try:
|
|
888
|
+
started = await tg.start_backfill(request, submission="new")
|
|
889
|
+
# Retain started.status.run.to_dict() for reopening and subsequent calls.
|
|
890
|
+
turn = await tg.prepare_backfill(BackfillPrepareContext.from_status(started.status))
|
|
891
|
+
if turn.batch is not None:
|
|
892
|
+
await destination.deliver_and_commit(turn.batch, turn.delivery)
|
|
893
|
+
await tg.acknowledge_backfill(turn.delivery)
|
|
894
|
+
finally:
|
|
895
|
+
await tg.close()
|
|
896
|
+
```
|
|
897
|
+
|
|
898
|
+
`destination.deliver_and_commit` is your application's durable receiver, not a tgdata
|
|
899
|
+
method. The full delivery reference identifies the obligation. Until acknowledged,
|
|
900
|
+
preparation replays the saved batch; a local wait returns without sleeping. The caller
|
|
901
|
+
schedules one source reader per group and keeps daily/backfill namespaces separate.
|
|
902
|
+
Collection identity does not automatically partition a backend; custom stores can use
|
|
903
|
+
separate namespaces in one database.
|
|
904
|
+
|
|
905
|
+
See [Backfill runs](docs/backfill_runs.md) for all six operations, status/error meanings,
|
|
906
|
+
exact retries, operator controls and recovery. Run the real SQLite receiver/restart
|
|
907
|
+
example without Telegram: `python examples/backfill_runs.py --demo`. The public API
|
|
908
|
+
passed offline fault tests and read-only live validation with Telethon 1.45.0, SQLite
|
|
909
|
+
and the selected test receiver. Your deployment must establish its own receiver
|
|
910
|
+
durability, reliable clock and single-reader ownership.
|
|
911
|
+
|
|
789
912
|
### Detecting & Downloading Media (photos / videos)
|
|
790
913
|
|
|
791
914
|
Every fetched message carries media-reference columns, so you can tell whether a
|
|
@@ -939,6 +1062,9 @@ await tg.run_with_event_loop()
|
|
|
939
1062
|
|
|
940
1063
|
### Account health events
|
|
941
1064
|
|
|
1065
|
+
This section describes the existing public read methods and `health_check()`.
|
|
1066
|
+
Verified temporary operations have the separate local view described below.
|
|
1067
|
+
|
|
942
1068
|
Every time Telegram says no to an account, tgdata tells you:
|
|
943
1069
|
- which verdict;
|
|
944
1070
|
- about what — the account, one kind of request, or one group;
|
|
@@ -1042,6 +1168,56 @@ error, so both now confirm with Telegram whether the session is logged in.
|
|
|
1042
1168
|
a level set on `telethon.client.users` during a call applies from the next
|
|
1043
1169
|
call.
|
|
1044
1170
|
|
|
1171
|
+
### Health for verified account operations
|
|
1172
|
+
|
|
1173
|
+
`tg.get_account_health(account_id)` returns this instance's local health snapshot
|
|
1174
|
+
for a **verified numeric account ID**, or `None` if it has not observed that account.
|
|
1175
|
+
The ID must be a positive integer, not a string or boolean. The call is synchronous:
|
|
1176
|
+
it does no connection, authentication or recovery. The returned dictionaries are
|
|
1177
|
+
independent copies, with the same fields as the health summary above.
|
|
1178
|
+
|
|
1179
|
+
`lookup_group`, `check_group_access` and `join_group` populate this account-owned view. Existing
|
|
1180
|
+
message reads and `health_check()` still use their existing health view; the views
|
|
1181
|
+
are not combined. A join result does not clear an earlier history-access failure.
|
|
1182
|
+
|
|
1183
|
+
Internally, an observation begins after a temporary client proves the expected
|
|
1184
|
+
account. Its event and retained state keep that numeric owner even if a cached
|
|
1185
|
+
identity changes later. The session name and account label are descriptive. Setup
|
|
1186
|
+
failures before proof remain errors to the caller, without inventing a condition
|
|
1187
|
+
for an unverified account. Forwarding those failures through another task or an
|
|
1188
|
+
exception wrapper preserves that neutral attribution. A separate Telegram error
|
|
1189
|
+
raised while handling one still reports normally; diagnostic classification of
|
|
1190
|
+
the original failure remains available.
|
|
1191
|
+
|
|
1192
|
+
Owned events use **`source: "rpc"`** for an actual health-related Telegram request
|
|
1193
|
+
failure, including one the operation subsequently handles. Local errors and their
|
|
1194
|
+
causes do not create these events. Recovery is conservative: successful work from
|
|
1195
|
+
an operation started before a condition was recorded cannot clear it; neither can
|
|
1196
|
+
the call that reported it. Owned event `request` and snapshot `waiting` keys name
|
|
1197
|
+
the logical request with its SDK namespace, such as `messages.GetHistoryRequest`
|
|
1198
|
+
or `channels.GetMessagesRequest`. Transport envelopes do not change that key;
|
|
1199
|
+
legacy health keeps its existing request names.
|
|
1200
|
+
|
|
1201
|
+
Waits require the matching request to succeed. A restriction requires that the
|
|
1202
|
+
previously refused request succeeds in a later operation with the same method.
|
|
1203
|
+
Another identity check cannot clear a history-read restriction. Group access
|
|
1204
|
+
requires explicit confirmation by the internal operation after an actual access
|
|
1205
|
+
result; self verification and metadata alone never imply access. The private
|
|
1206
|
+
[operation contract](docs/account_operations.md#owned-health-composition-stage-2)
|
|
1207
|
+
describes how request evidence is captured.
|
|
1208
|
+
|
|
1209
|
+
Owned state is updated before the operation returns. Both synchronous and async
|
|
1210
|
+
health callbacks run in separate tasks **after the source's disconnect attempt
|
|
1211
|
+
finishes**. A callback may run after the result or error reaches its caller. Its
|
|
1212
|
+
ordinary errors or cancellation cannot change that outcome; callback re-entry is
|
|
1213
|
+
suppressed as above. Unequal cleanup durations can make callbacks arrive in a
|
|
1214
|
+
different order from the observations; use event times and the local snapshot to
|
|
1215
|
+
interpret state. Notification delivery is not durable or guaranteed at process
|
|
1216
|
+
exit. Finish ongoing operations before `await tg.close()`, which cooperatively
|
|
1217
|
+
cancels and awaits pending owned notifications. A callback may itself call close.
|
|
1218
|
+
These timing rules apply to the new owned path; the existing public health callback
|
|
1219
|
+
timing remains as documented above.
|
|
1220
|
+
|
|
1045
1221
|
### Performance Tips
|
|
1046
1222
|
|
|
1047
1223
|
- The client authenticates once and keeps a single **persistent connection**,
|
|
@@ -1054,7 +1230,6 @@ error, so both now confirm with Telegram whether the session is logged in.
|
|
|
1054
1230
|
## Requirements
|
|
1055
1231
|
|
|
1056
1232
|
- Python 3.7+
|
|
1057
|
-
- Telethon 1.45.0 or newer within 1.x (verification target: 1.45.0)
|
|
1058
1233
|
- Telegram API credentials (not bot tokens)
|
|
1059
1234
|
- Group/channel membership
|
|
1060
1235
|
|