etlplus 0.5.2__py3-none-any.whl → 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlplus/api/README.md +24 -26
- etlplus/cli/commands.py +924 -0
- etlplus/cli/constants.py +71 -0
- etlplus/cli/handlers.py +369 -484
- etlplus/cli/io.py +336 -0
- etlplus/cli/main.py +16 -418
- etlplus/cli/options.py +49 -0
- etlplus/cli/state.py +336 -0
- etlplus/cli/types.py +33 -0
- etlplus/database/__init__.py +44 -0
- etlplus/database/ddl.py +319 -0
- etlplus/database/engine.py +151 -0
- etlplus/database/orm.py +354 -0
- etlplus/database/schema.py +274 -0
- etlplus/database/types.py +33 -0
- etlplus/enums.py +51 -1
- etlplus/load.py +1 -1
- etlplus/run.py +2 -4
- etlplus/types.py +5 -0
- etlplus/utils.py +1 -32
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/METADATA +84 -40
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/RECORD +26 -16
- etlplus/cli/app.py +0 -1367
- etlplus/ddl.py +0 -197
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/WHEEL +0 -0
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/entry_points.txt +0 -0
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/licenses/LICENSE +0 -0
- {etlplus-0.5.2.dist-info → etlplus-0.9.1.dist-info}/top_level.txt +0 -0
etlplus/api/README.md
CHANGED
|
@@ -7,8 +7,8 @@ paginated REST endpoints.
|
|
|
7
7
|
- Supports page-, offset-, and cursor-based pagination via `PaginationConfig`
|
|
8
8
|
- Simple bearer-auth credentials via `EndpointCredentialsBearer`
|
|
9
9
|
- Convenience helpers to extract records from nested JSON payloads
|
|
10
|
-
- Returns
|
|
11
|
-
|
|
10
|
+
- Returns paginated JSON payloads (lists of record dictionaries) consistent with the rest of the
|
|
11
|
+
library.
|
|
12
12
|
|
|
13
13
|
Back to project overview: see the top-level [README](../../README.md).
|
|
14
14
|
|
|
@@ -29,7 +29,6 @@ import requests
|
|
|
29
29
|
from etlplus.api import (
|
|
30
30
|
EndpointClient,
|
|
31
31
|
EndpointCredentialsBearer,
|
|
32
|
-
JSONRecords,
|
|
33
32
|
)
|
|
34
33
|
|
|
35
34
|
auth = EndpointCredentialsBearer(
|
|
@@ -54,12 +53,12 @@ client = EndpointClient(
|
|
|
54
53
|
|
|
55
54
|
# Page-based pagination
|
|
56
55
|
pg: PaginationConfig = {"type": "page", "page_size": 100}
|
|
57
|
-
rows
|
|
56
|
+
rows = client.paginate("list", pagination=pg)
|
|
58
57
|
for row in rows:
|
|
59
58
|
print(row)
|
|
60
59
|
```
|
|
61
60
|
|
|
62
|
-
### Overriding
|
|
61
|
+
### Overriding Rate Limits Per Call
|
|
63
62
|
|
|
64
63
|
When a client is constructed with ``rate_limit`` metadata you can still tweak the pacing for
|
|
65
64
|
individual calls by passing ``rate_limit_overrides`` to ``paginate``/``paginate_iter``. The
|
|
@@ -101,10 +100,10 @@ If the API responds like this:
|
|
|
101
100
|
|
|
102
101
|
If the response is a list at the top level, you can omit `records_path`.
|
|
103
102
|
|
|
104
|
-
## Cursor-
|
|
103
|
+
## Cursor-Based Pagination Example
|
|
105
104
|
|
|
106
105
|
```python
|
|
107
|
-
from etlplus.api import EndpointClient, PaginationConfig
|
|
106
|
+
from etlplus.api import EndpointClient, PaginationConfig
|
|
108
107
|
|
|
109
108
|
client = EndpointClient(
|
|
110
109
|
base_url="https://api.example.com/v1",
|
|
@@ -125,7 +124,7 @@ pg: PaginationConfig = {
|
|
|
125
124
|
# "start_cursor": "abc123",
|
|
126
125
|
}
|
|
127
126
|
|
|
128
|
-
rows
|
|
127
|
+
rows = client.paginate("list", pagination=pg)
|
|
129
128
|
for row in rows:
|
|
130
129
|
process(row)
|
|
131
130
|
```
|
|
@@ -192,36 +191,35 @@ client = EndpointClient(
|
|
|
192
191
|
providers can fall back to their own defaults. If you already possess a static token, attach it to a
|
|
193
192
|
`requests.Session` manually rather than instantiating `EndpointCredentialsBearer`.
|
|
194
193
|
|
|
195
|
-
## Errors and
|
|
194
|
+
## Errors and Rate Limiting
|
|
196
195
|
|
|
197
196
|
- Errors: `ApiRequestError`, `ApiAuthError`, and `PaginationError` (in `etlplus/api/errors.py`)
|
|
198
197
|
include an `as_dict()` helper for structured logs.
|
|
199
|
-
- Rate limiting: `RateLimiter`
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
`
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
`
|
|
210
|
-
|
|
198
|
+
- Rate limiting: `RateLimiter` (in `etlplus/api/rate_limiting/rate_limiter.py`) derives fixed sleeps
|
|
199
|
+
or `max_per_sec` windows. The paginator now builds a `RateLimiter` whenever the effective delay
|
|
200
|
+
comes from `rate_limit`/`rate_limit_overrides`, so each page fetch sleeps before making another
|
|
201
|
+
HTTP call. Passing `rate_limit_overrides` to `paginate*` lets you momentarily speed up or slow
|
|
202
|
+
down a single request without mutating the client-wide defaults.
|
|
203
|
+
|
|
204
|
+
## Types and Transport
|
|
205
|
+
|
|
206
|
+
- Types: pagination config helpers live in `etlplus/api/pagination/paginator.py`; retry helpers
|
|
207
|
+
(including `RetryPolicy`) live in `etlplus/api/retry_manager.py`; rate-limit helpers live in
|
|
208
|
+
`etlplus/api/rate_limiting/rate_limiter.py`. These are all re-exported from `etlplus.api` for
|
|
209
|
+
convenience.
|
|
211
210
|
- Transport/session: `etlplus/api/transport.py` contains the HTTP adapter helpers and
|
|
212
211
|
`etlplus/api/request_manager.py` wraps `requests` sessions plus retry orchestration. Advanced
|
|
213
212
|
users may consult those modules to adapt behavior.
|
|
214
213
|
|
|
215
|
-
## Supporting
|
|
214
|
+
## Supporting Modules
|
|
216
215
|
|
|
217
216
|
- `etlplus.api.types` collects friendly aliases such as `Headers`, `Params`, `Url`, and
|
|
218
217
|
`RateLimitOverrides` (whose values accept numeric override inputs) so endpoint helpers share the
|
|
219
218
|
same type vocabulary.
|
|
220
|
-
- `etlplus.utils` exposes lightweight helpers used across the project, including
|
|
221
|
-
|
|
222
|
-
`to_positive_int`, etc.).
|
|
219
|
+
- `etlplus.utils` exposes lightweight helpers used across the project, including `print_json` and
|
|
220
|
+
numeric coercion utilities (`to_float`, `to_positive_int`, etc.).
|
|
223
221
|
|
|
224
|
-
## Minimal
|
|
222
|
+
## Minimal Contract
|
|
225
223
|
|
|
226
224
|
- Inputs
|
|
227
225
|
- `base_url: str`, `endpoints: dict[str, str]`
|