hadoku-scrape-sdk 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hadoku_scrape_sdk-0.1.3/LICENSE +15 -0
- hadoku_scrape_sdk-0.1.3/PKG-INFO +103 -0
- hadoku_scrape_sdk-0.1.3/README.md +76 -0
- hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/PKG-INFO +103 -0
- hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/SOURCES.txt +60 -0
- hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/dependency_links.txt +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/requires.txt +18 -0
- hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/top_level.txt +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/auth.py +181 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/errors.py +296 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/fire_and_forget.py +159 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/lock_release.py +30 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/whoami.py +205 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/models/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/models/responses.py +25 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/auth/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/auth/cookies.py +317 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/__init__.py +23 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/cdpfree.py +560 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/linkedin.py +25 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/manager.py +1002 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/preflight.py +556 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/session.py +405 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/stealth.py +254 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/config.py +45 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/engine.py +378 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/execution_log.py +726 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/migrations/001_execution_log.sql +28 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/migrations/002_run_pacing.sql +20 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/migrations/003_execution_log_owner.sql +15 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/__init__.py +8 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/aggregator.py +107 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/discord_notify.py +112 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/logging.py +95 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/paths.py +28 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/plugins/__init__.py +290 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/plugins/__main__.py +28 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/plugins/manifest.py +115 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/__init__.py +12 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/base.py +125 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/client.py +284 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/escalation.py +270 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/__init__.py +36 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/events.py +170 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/gate.py +50 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/pacing.py +227 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/reaper.py +98 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/base.py +274 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/cloudflare_errors.py +89 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/cloudflare_kv.py +245 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/config.py +40 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/testing/__init__.py +1 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/testing/pytest_plugin.py +265 -0
- hadoku_scrape_sdk-0.1.3/hadoku_sdk/testing/unit.py +40 -0
- hadoku_scrape_sdk-0.1.3/pyproject.toml +131 -0
- hadoku_scrape_sdk-0.1.3/setup.cfg +4 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
ISC License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 WolffM
|
|
4
|
+
|
|
5
|
+
Permission to use, copy, modify, and/or distribute this software for any
|
|
6
|
+
purpose with or without fee is hereby granted, provided that the above
|
|
7
|
+
copyright notice and this permission notice appear in all copies.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
10
|
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
11
|
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
12
|
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
13
|
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
14
|
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
15
|
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hadoku-scrape-sdk
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: The runtime hadoku-scrape plugins build on: plugin contract, browser stack, run log, events, test floor
|
|
5
|
+
License-Expression: ISC
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: browser-cookie3<1.0,>=0.19.0
|
|
10
|
+
Requires-Dist: fastapi<1.0,>=0.109.0
|
|
11
|
+
Requires-Dist: hadoku-execlog>=0.1.0
|
|
12
|
+
Requires-Dist: httpx[socks]<1.0,>=0.27.0
|
|
13
|
+
Requires-Dist: patchright<2.0,>=1.49.0
|
|
14
|
+
Requires-Dist: playwright<2.0,>=1.62.0
|
|
15
|
+
Requires-Dist: pydantic<3.0,>=2.0
|
|
16
|
+
Requires-Dist: pyvirtualdisplay<4.0,>=3.0
|
|
17
|
+
Requires-Dist: structlog<25.0,>=24.0.0
|
|
18
|
+
Requires-Dist: tenacity<9.0,>=8.2.0
|
|
19
|
+
Provides-Extra: testing
|
|
20
|
+
Requires-Dist: pytest==8.4.2; extra == "testing"
|
|
21
|
+
Requires-Dist: pytest-asyncio<2,>=1.2; extra == "testing"
|
|
22
|
+
Requires-Dist: pytest-timeout<3.0,>=2.2.0; extra == "testing"
|
|
23
|
+
Requires-Dist: respx<1.0,>=0.21.0; extra == "testing"
|
|
24
|
+
Requires-Dist: httpx2<3.0,>=2.9; extra == "testing"
|
|
25
|
+
Requires-Dist: typer<1.0,>=0.15.0; extra == "testing"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# hadoku-scrape-sdk
|
|
29
|
+
|
|
30
|
+
The runtime that hadoku-scrape plugins are built on. A plugin is a scraper that
|
|
31
|
+
lives in its own repo and runs inside the hadoku-scrape service. This package is
|
|
32
|
+
everything such a plugin imports, and nothing of the service around it.
|
|
33
|
+
|
|
34
|
+
```sh
|
|
35
|
+
pip install hadoku-scrape-sdk # runtime
|
|
36
|
+
pip install 'hadoku-scrape-sdk[testing]' # + the pinned test stack and test floor
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## What's in it
|
|
40
|
+
|
|
41
|
+
| Module | What it gives a plugin |
|
|
42
|
+
|---|---|
|
|
43
|
+
| `hadoku_sdk.plugins` | `ScraperPlugin`, `UrlScraper`: the contract. `python -m hadoku_sdk.plugins` checks every enabled plugin loads |
|
|
44
|
+
| `hadoku_sdk.db.engine`, `.db.execution_log` | The SQLite engine (`init_db`, `get_connection`) and `execution_run`, the run log every scrape writes through |
|
|
45
|
+
| `hadoku_sdk.runtime` | `Severity` and events (`ctx.event(...)`), run pacing, the run gate |
|
|
46
|
+
| `hadoku_sdk.browser` | Browser manager (Patchright, persistent profiles, stealth), sessions, preflight checks |
|
|
47
|
+
| `hadoku_sdk.proxy` | HTTP client with proxy escalation |
|
|
48
|
+
| `hadoku_sdk.auth.cookies` | Browser-cookie import |
|
|
49
|
+
| `hadoku_sdk.api.middleware` | Auth dependencies (`require_service`, ...), error types, `spawn_tracked` for fire-and-forget routes |
|
|
50
|
+
| `hadoku_sdk.scrapers.base` | `BaseScraper`, `ScraperResult` for URL-dispatch scrapers |
|
|
51
|
+
| `hadoku_sdk.observability.logging` | `get_logger` (structlog) |
|
|
52
|
+
| `hadoku_sdk.testing` | The autouse test floor: an isolated DB per test, no real browsers |
|
|
53
|
+
|
|
54
|
+
## A plugin
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
# my_plugins/thing/plugin.py
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
from hadoku_sdk.plugins import ScraperPlugin
|
|
60
|
+
|
|
61
|
+
PLUGIN = ScraperPlugin(
|
|
62
|
+
name="thing",
|
|
63
|
+
routers=[router], # FastAPI, prefix /api/v1/<yours>
|
|
64
|
+
cli={"thing": thing_app}, # Typer, mounted as `hs thing`
|
|
65
|
+
migrations=Path(__file__).parent / "migrations", # NNN_*.sql
|
|
66
|
+
on_startup=[seed],
|
|
67
|
+
)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
```toml
|
|
71
|
+
# pyproject.toml
|
|
72
|
+
[project.entry-points."hadoku_scrape.plugins"]
|
|
73
|
+
thing = "my_plugins.thing.plugin:PLUGIN"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
The host's `plugins.toml` decides which installed plugins actually load.
|
|
77
|
+
|
|
78
|
+
## Tests
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
# tests/conftest.py
|
|
82
|
+
import os
|
|
83
|
+
from pathlib import Path
|
|
84
|
+
os.environ["HADOKU_PLUGINS_MANIFEST"] = str(Path(__file__).parent / "resources" / "plugins.toml")
|
|
85
|
+
pytest_plugins = ["hadoku_sdk.testing.pytest_plugin"]
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
# tests/unit/conftest.py
|
|
90
|
+
from hadoku_sdk.testing.unit import _instant_sleep, _silence_discord_notify # noqa: F401
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Rules a plugin follows
|
|
94
|
+
|
|
95
|
+
- Notifications go through `ctx.event(name, payload, severity=...)` on the run
|
|
96
|
+
context, never a direct Discord call. The host routes them.
|
|
97
|
+
- Timestamps are UTC. Never `datetime.utcnow()`.
|
|
98
|
+
- Never log credentials, cookies or API keys.
|
|
99
|
+
- No lazy imports inside functions that run on background threads.
|
|
100
|
+
- Drive browsers through `hadoku_sdk.browser.manager`. Never hardcode locale or
|
|
101
|
+
timezone.
|
|
102
|
+
|
|
103
|
+
License: ISC.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# hadoku-scrape-sdk
|
|
2
|
+
|
|
3
|
+
The runtime that hadoku-scrape plugins are built on. A plugin is a scraper that
|
|
4
|
+
lives in its own repo and runs inside the hadoku-scrape service. This package is
|
|
5
|
+
everything such a plugin imports, and nothing of the service around it.
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
pip install hadoku-scrape-sdk # runtime
|
|
9
|
+
pip install 'hadoku-scrape-sdk[testing]' # + the pinned test stack and test floor
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## What's in it
|
|
13
|
+
|
|
14
|
+
| Module | What it gives a plugin |
|
|
15
|
+
|---|---|
|
|
16
|
+
| `hadoku_sdk.plugins` | `ScraperPlugin`, `UrlScraper`: the contract. `python -m hadoku_sdk.plugins` checks every enabled plugin loads |
|
|
17
|
+
| `hadoku_sdk.db.engine`, `.db.execution_log` | The SQLite engine (`init_db`, `get_connection`) and `execution_run`, the run log every scrape writes through |
|
|
18
|
+
| `hadoku_sdk.runtime` | `Severity` and events (`ctx.event(...)`), run pacing, the run gate |
|
|
19
|
+
| `hadoku_sdk.browser` | Browser manager (Patchright, persistent profiles, stealth), sessions, preflight checks |
|
|
20
|
+
| `hadoku_sdk.proxy` | HTTP client with proxy escalation |
|
|
21
|
+
| `hadoku_sdk.auth.cookies` | Browser-cookie import |
|
|
22
|
+
| `hadoku_sdk.api.middleware` | Auth dependencies (`require_service`, ...), error types, `spawn_tracked` for fire-and-forget routes |
|
|
23
|
+
| `hadoku_sdk.scrapers.base` | `BaseScraper`, `ScraperResult` for URL-dispatch scrapers |
|
|
24
|
+
| `hadoku_sdk.observability.logging` | `get_logger` (structlog) |
|
|
25
|
+
| `hadoku_sdk.testing` | The autouse test floor: an isolated DB per test, no real browsers |
|
|
26
|
+
|
|
27
|
+
## A plugin
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
# my_plugins/thing/plugin.py
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from hadoku_sdk.plugins import ScraperPlugin
|
|
33
|
+
|
|
34
|
+
PLUGIN = ScraperPlugin(
|
|
35
|
+
name="thing",
|
|
36
|
+
routers=[router], # FastAPI, prefix /api/v1/<yours>
|
|
37
|
+
cli={"thing": thing_app}, # Typer, mounted as `hs thing`
|
|
38
|
+
migrations=Path(__file__).parent / "migrations", # NNN_*.sql
|
|
39
|
+
on_startup=[seed],
|
|
40
|
+
)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
```toml
|
|
44
|
+
# pyproject.toml
|
|
45
|
+
[project.entry-points."hadoku_scrape.plugins"]
|
|
46
|
+
thing = "my_plugins.thing.plugin:PLUGIN"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The host's `plugins.toml` decides which installed plugins actually load.
|
|
50
|
+
|
|
51
|
+
## Tests
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
# tests/conftest.py
|
|
55
|
+
import os
|
|
56
|
+
from pathlib import Path
|
|
57
|
+
os.environ["HADOKU_PLUGINS_MANIFEST"] = str(Path(__file__).parent / "resources" / "plugins.toml")
|
|
58
|
+
pytest_plugins = ["hadoku_sdk.testing.pytest_plugin"]
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
# tests/unit/conftest.py
|
|
63
|
+
from hadoku_sdk.testing.unit import _instant_sleep, _silence_discord_notify # noqa: F401
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Rules a plugin follows
|
|
67
|
+
|
|
68
|
+
- Notifications go through `ctx.event(name, payload, severity=...)` on the run
|
|
69
|
+
context, never a direct Discord call. The host routes them.
|
|
70
|
+
- Timestamps are UTC. Never `datetime.utcnow()`.
|
|
71
|
+
- Never log credentials, cookies or API keys.
|
|
72
|
+
- No lazy imports inside functions that run on background threads.
|
|
73
|
+
- Drive browsers through `hadoku_sdk.browser.manager`. Never hardcode locale or
|
|
74
|
+
timezone.
|
|
75
|
+
|
|
76
|
+
License: ISC.
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hadoku-scrape-sdk
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: The runtime hadoku-scrape plugins build on: plugin contract, browser stack, run log, events, test floor
|
|
5
|
+
License-Expression: ISC
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: browser-cookie3<1.0,>=0.19.0
|
|
10
|
+
Requires-Dist: fastapi<1.0,>=0.109.0
|
|
11
|
+
Requires-Dist: hadoku-execlog>=0.1.0
|
|
12
|
+
Requires-Dist: httpx[socks]<1.0,>=0.27.0
|
|
13
|
+
Requires-Dist: patchright<2.0,>=1.49.0
|
|
14
|
+
Requires-Dist: playwright<2.0,>=1.62.0
|
|
15
|
+
Requires-Dist: pydantic<3.0,>=2.0
|
|
16
|
+
Requires-Dist: pyvirtualdisplay<4.0,>=3.0
|
|
17
|
+
Requires-Dist: structlog<25.0,>=24.0.0
|
|
18
|
+
Requires-Dist: tenacity<9.0,>=8.2.0
|
|
19
|
+
Provides-Extra: testing
|
|
20
|
+
Requires-Dist: pytest==8.4.2; extra == "testing"
|
|
21
|
+
Requires-Dist: pytest-asyncio<2,>=1.2; extra == "testing"
|
|
22
|
+
Requires-Dist: pytest-timeout<3.0,>=2.2.0; extra == "testing"
|
|
23
|
+
Requires-Dist: respx<1.0,>=0.21.0; extra == "testing"
|
|
24
|
+
Requires-Dist: httpx2<3.0,>=2.9; extra == "testing"
|
|
25
|
+
Requires-Dist: typer<1.0,>=0.15.0; extra == "testing"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# hadoku-scrape-sdk
|
|
29
|
+
|
|
30
|
+
The runtime that hadoku-scrape plugins are built on. A plugin is a scraper that
|
|
31
|
+
lives in its own repo and runs inside the hadoku-scrape service. This package is
|
|
32
|
+
everything such a plugin imports, and nothing of the service around it.
|
|
33
|
+
|
|
34
|
+
```sh
|
|
35
|
+
pip install hadoku-scrape-sdk # runtime
|
|
36
|
+
pip install 'hadoku-scrape-sdk[testing]' # + the pinned test stack and test floor
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## What's in it
|
|
40
|
+
|
|
41
|
+
| Module | What it gives a plugin |
|
|
42
|
+
|---|---|
|
|
43
|
+
| `hadoku_sdk.plugins` | `ScraperPlugin`, `UrlScraper`: the contract. `python -m hadoku_sdk.plugins` checks every enabled plugin loads |
|
|
44
|
+
| `hadoku_sdk.db.engine`, `.db.execution_log` | The SQLite engine (`init_db`, `get_connection`) and `execution_run`, the run log every scrape writes through |
|
|
45
|
+
| `hadoku_sdk.runtime` | `Severity` and events (`ctx.event(...)`), run pacing, the run gate |
|
|
46
|
+
| `hadoku_sdk.browser` | Browser manager (Patchright, persistent profiles, stealth), sessions, preflight checks |
|
|
47
|
+
| `hadoku_sdk.proxy` | HTTP client with proxy escalation |
|
|
48
|
+
| `hadoku_sdk.auth.cookies` | Browser-cookie import |
|
|
49
|
+
| `hadoku_sdk.api.middleware` | Auth dependencies (`require_service`, ...), error types, `spawn_tracked` for fire-and-forget routes |
|
|
50
|
+
| `hadoku_sdk.scrapers.base` | `BaseScraper`, `ScraperResult` for URL-dispatch scrapers |
|
|
51
|
+
| `hadoku_sdk.observability.logging` | `get_logger` (structlog) |
|
|
52
|
+
| `hadoku_sdk.testing` | The autouse test floor: an isolated DB per test, no real browsers |
|
|
53
|
+
|
|
54
|
+
## A plugin
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
# my_plugins/thing/plugin.py
|
|
58
|
+
from pathlib import Path
|
|
59
|
+
from hadoku_sdk.plugins import ScraperPlugin
|
|
60
|
+
|
|
61
|
+
PLUGIN = ScraperPlugin(
|
|
62
|
+
name="thing",
|
|
63
|
+
routers=[router], # FastAPI, prefix /api/v1/<yours>
|
|
64
|
+
cli={"thing": thing_app}, # Typer, mounted as `hs thing`
|
|
65
|
+
migrations=Path(__file__).parent / "migrations", # NNN_*.sql
|
|
66
|
+
on_startup=[seed],
|
|
67
|
+
)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
```toml
|
|
71
|
+
# pyproject.toml
|
|
72
|
+
[project.entry-points."hadoku_scrape.plugins"]
|
|
73
|
+
thing = "my_plugins.thing.plugin:PLUGIN"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
The host's `plugins.toml` decides which installed plugins actually load.
|
|
77
|
+
|
|
78
|
+
## Tests
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
# tests/conftest.py
|
|
82
|
+
import os
|
|
83
|
+
from pathlib import Path
|
|
84
|
+
os.environ["HADOKU_PLUGINS_MANIFEST"] = str(Path(__file__).parent / "resources" / "plugins.toml")
|
|
85
|
+
pytest_plugins = ["hadoku_sdk.testing.pytest_plugin"]
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
# tests/unit/conftest.py
|
|
90
|
+
from hadoku_sdk.testing.unit import _instant_sleep, _silence_discord_notify # noqa: F401
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Rules a plugin follows
|
|
94
|
+
|
|
95
|
+
- Notifications go through `ctx.event(name, payload, severity=...)` on the run
|
|
96
|
+
context, never a direct Discord call. The host routes them.
|
|
97
|
+
- Timestamps are UTC. Never `datetime.utcnow()`.
|
|
98
|
+
- Never log credentials, cookies or API keys.
|
|
99
|
+
- No lazy imports inside functions that run on background threads.
|
|
100
|
+
- Drive browsers through `hadoku_sdk.browser.manager`. Never hardcode locale or
|
|
101
|
+
timezone.
|
|
102
|
+
|
|
103
|
+
License: ISC.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
hadoku_scrape_sdk.egg-info/PKG-INFO
|
|
5
|
+
hadoku_scrape_sdk.egg-info/SOURCES.txt
|
|
6
|
+
hadoku_scrape_sdk.egg-info/dependency_links.txt
|
|
7
|
+
hadoku_scrape_sdk.egg-info/requires.txt
|
|
8
|
+
hadoku_scrape_sdk.egg-info/top_level.txt
|
|
9
|
+
hadoku_sdk/__init__.py
|
|
10
|
+
hadoku_sdk/config.py
|
|
11
|
+
hadoku_sdk/paths.py
|
|
12
|
+
hadoku_sdk/api/__init__.py
|
|
13
|
+
hadoku_sdk/api/middleware/__init__.py
|
|
14
|
+
hadoku_sdk/api/middleware/auth.py
|
|
15
|
+
hadoku_sdk/api/middleware/errors.py
|
|
16
|
+
hadoku_sdk/api/middleware/fire_and_forget.py
|
|
17
|
+
hadoku_sdk/api/middleware/lock_release.py
|
|
18
|
+
hadoku_sdk/api/middleware/whoami.py
|
|
19
|
+
hadoku_sdk/api/models/__init__.py
|
|
20
|
+
hadoku_sdk/api/models/responses.py
|
|
21
|
+
hadoku_sdk/auth/__init__.py
|
|
22
|
+
hadoku_sdk/auth/cookies.py
|
|
23
|
+
hadoku_sdk/browser/__init__.py
|
|
24
|
+
hadoku_sdk/browser/cdpfree.py
|
|
25
|
+
hadoku_sdk/browser/linkedin.py
|
|
26
|
+
hadoku_sdk/browser/manager.py
|
|
27
|
+
hadoku_sdk/browser/preflight.py
|
|
28
|
+
hadoku_sdk/browser/session.py
|
|
29
|
+
hadoku_sdk/browser/stealth.py
|
|
30
|
+
hadoku_sdk/db/__init__.py
|
|
31
|
+
hadoku_sdk/db/engine.py
|
|
32
|
+
hadoku_sdk/db/execution_log.py
|
|
33
|
+
hadoku_sdk/db/migrations/001_execution_log.sql
|
|
34
|
+
hadoku_sdk/db/migrations/002_run_pacing.sql
|
|
35
|
+
hadoku_sdk/db/migrations/003_execution_log_owner.sql
|
|
36
|
+
hadoku_sdk/observability/__init__.py
|
|
37
|
+
hadoku_sdk/observability/aggregator.py
|
|
38
|
+
hadoku_sdk/observability/discord_notify.py
|
|
39
|
+
hadoku_sdk/observability/logging.py
|
|
40
|
+
hadoku_sdk/plugins/__init__.py
|
|
41
|
+
hadoku_sdk/plugins/__main__.py
|
|
42
|
+
hadoku_sdk/plugins/manifest.py
|
|
43
|
+
hadoku_sdk/proxy/__init__.py
|
|
44
|
+
hadoku_sdk/proxy/base.py
|
|
45
|
+
hadoku_sdk/proxy/client.py
|
|
46
|
+
hadoku_sdk/proxy/escalation.py
|
|
47
|
+
hadoku_sdk/runtime/__init__.py
|
|
48
|
+
hadoku_sdk/runtime/events.py
|
|
49
|
+
hadoku_sdk/runtime/gate.py
|
|
50
|
+
hadoku_sdk/runtime/pacing.py
|
|
51
|
+
hadoku_sdk/runtime/reaper.py
|
|
52
|
+
hadoku_sdk/scrapers/__init__.py
|
|
53
|
+
hadoku_sdk/scrapers/base.py
|
|
54
|
+
hadoku_sdk/scrapers/utils/__init__.py
|
|
55
|
+
hadoku_sdk/scrapers/utils/cloudflare_errors.py
|
|
56
|
+
hadoku_sdk/scrapers/utils/cloudflare_kv.py
|
|
57
|
+
hadoku_sdk/scrapers/utils/config.py
|
|
58
|
+
hadoku_sdk/testing/__init__.py
|
|
59
|
+
hadoku_sdk/testing/pytest_plugin.py
|
|
60
|
+
hadoku_sdk/testing/unit.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
browser-cookie3<1.0,>=0.19.0
|
|
2
|
+
fastapi<1.0,>=0.109.0
|
|
3
|
+
hadoku-execlog>=0.1.0
|
|
4
|
+
httpx[socks]<1.0,>=0.27.0
|
|
5
|
+
patchright<2.0,>=1.49.0
|
|
6
|
+
playwright<2.0,>=1.62.0
|
|
7
|
+
pydantic<3.0,>=2.0
|
|
8
|
+
pyvirtualdisplay<4.0,>=3.0
|
|
9
|
+
structlog<25.0,>=24.0.0
|
|
10
|
+
tenacity<9.0,>=8.2.0
|
|
11
|
+
|
|
12
|
+
[testing]
|
|
13
|
+
pytest==8.4.2
|
|
14
|
+
pytest-asyncio<2,>=1.2
|
|
15
|
+
pytest-timeout<3.0,>=2.2.0
|
|
16
|
+
respx<1.0,>=0.21.0
|
|
17
|
+
httpx2<3.0,>=2.9
|
|
18
|
+
typer<1.0,>=0.15.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
hadoku_sdk
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""hadoku-scrape SDK: what a scraper plugin builds on, without the service around it."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Part of hadoku-scrape-sdk."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Part of hadoku-scrape-sdk."""
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Tier-aware API authentication.
|
|
2
|
+
|
|
3
|
+
The edge in front of the service resolves the caller's session/key and
|
|
4
|
+
injects ``X-User-Key`` on the proxied request. The backend
|
|
5
|
+
holds **no key arrays of its own** — it delegates tier resolution to the
|
|
6
|
+
platform's authoritative ``/session/whoami`` endpoint (see
|
|
7
|
+
:mod:`hadoku_sdk.api.middleware.whoami`), cached per key. This works for
|
|
8
|
+
both edge-tunnelled callers and the direct service-to-service callers
|
|
9
|
+
that bypass the edge.
|
|
10
|
+
|
|
11
|
+
Tiers:
|
|
12
|
+
- ``admin`` operator key, full access
|
|
13
|
+
- ``service`` internal sibling-service key — full access EXCEPT routes explicitly
|
|
14
|
+
admin-only, and only for a key on the ``scraper-service``
|
|
15
|
+
access list
|
|
16
|
+
- ``friend`` trusted human, read-only observability access
|
|
17
|
+
- ``public`` no key / unrecognised key
|
|
18
|
+
|
|
19
|
+
Per-route deps name the *minimum* tier accepted; admin is implicit.
|
|
20
|
+
|
|
21
|
+
- ``require_admin`` admin only
|
|
22
|
+
- ``require_service`` service or higher AND on ``scraper-service`` (default
|
|
23
|
+
for routes that mutate state / consume resources /
|
|
24
|
+
call external APIs). Service tier is every service
|
|
25
|
+
key on the platform, and people hold them too; the list
|
|
26
|
+
names the callers that drive the scraper. Admin is
|
|
27
|
+
not exempt: the operator is on the list.
|
|
28
|
+
- ``require_friend`` friend or higher (dashboard / observability /
|
|
29
|
+
contributor-friendly trigger endpoints — anything
|
|
30
|
+
a trusted human should be able to hit)
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
from enum import Enum
|
|
37
|
+
|
|
38
|
+
from fastapi import Depends, Request
|
|
39
|
+
|
|
40
|
+
from hadoku_sdk.api.middleware.errors import AuthenticationError, AuthorizationError
|
|
41
|
+
from hadoku_sdk.api.middleware.whoami import resolve_identity_from_key
|
|
42
|
+
from hadoku_sdk.observability.logging import get_logger
|
|
43
|
+
|
|
44
|
+
log = get_logger("api.auth")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Tier(str, Enum):
|
|
48
|
+
"""The tier ladder. Mirrors `_VALID_TIERS` in `whoami.py`: if a tier exists
|
|
49
|
+
there and not here, `_resolve_auth` raises `ValueError` on it, which is a 500
|
|
50
|
+
rather than a 403. Edge tier names beyond these are mapped onto them with
|
|
51
|
+
`whoami.alias_tier`.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
ADMIN = "admin"
|
|
55
|
+
MEMBER = "member"
|
|
56
|
+
SERVICE = "service"
|
|
57
|
+
FRIEND = "friend"
|
|
58
|
+
PUBLIC = "public"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# LOW to HIGH. Gates compare RANK, never identity or membership: `tier is Tier.ADMIN`
|
|
62
|
+
# and `tier not in (Tier.ADMIN, Tier.SERVICE)` both lock out any tier that sits ABOVE
|
|
63
|
+
# the one being named, which is the opposite of what a minimum-tier gate means.
|
|
64
|
+
_RANK: dict[Tier, int] = {
|
|
65
|
+
Tier.PUBLIC: 0,
|
|
66
|
+
Tier.FRIEND: 1,
|
|
67
|
+
Tier.SERVICE: 2,
|
|
68
|
+
Tier.MEMBER: 3,
|
|
69
|
+
Tier.ADMIN: 4,
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def tier_at_least(tier: Tier, minimum: Tier) -> bool:
|
|
74
|
+
"""True when `tier` sits at or above `minimum` on the ladder."""
|
|
75
|
+
return _RANK.get(tier, -1) >= _RANK[minimum]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class AuthContext:
|
|
80
|
+
"""Resolved caller identity. Yielded by the require_* deps so route
|
|
81
|
+
handlers can stamp audit fields without ever touching the raw key."""
|
|
82
|
+
|
|
83
|
+
tier: Tier
|
|
84
|
+
masked_key: str
|
|
85
|
+
#: The platform's stable per-user UUID, straight from whoami — which the
|
|
86
|
+
#: resolver has to call anyway to get the tier. Durable across sessions and
|
|
87
|
+
#: devices, so it is the right key for anything stored per person. None
|
|
88
|
+
#: when whoami did not supply one (anonymous, transient failure, or a test
|
|
89
|
+
#: override), which callers must handle rather than assume.
|
|
90
|
+
user_id: str | None = None
|
|
91
|
+
#: On the ``scraper-service`` access list (whoami answers it per key).
|
|
92
|
+
member: bool = False
|
|
93
|
+
|
|
94
|
+
def __str__(self) -> str:
|
|
95
|
+
# The userId, not the masked key: this is what lands in an f-string,
|
|
96
|
+
# and from there in whatever the f-string is written to.
|
|
97
|
+
return f"{self.tier.value}:{self.user_id or 'no-user'}"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _mask(key: str) -> str:
|
|
101
|
+
return f"{key[:4]}…{key[-2:]}" if len(key) > 8 else "****"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _extract_key(request: Request) -> str | None:
|
|
105
|
+
"""Read the caller key from ``X-User-Key``. Bearer tokens are no longer
|
|
106
|
+
accepted at the backend — the edge translates session/cookie/Bearer
|
|
107
|
+
forms into ``X-User-Key`` before proxying."""
|
|
108
|
+
raw = request.headers.get("X-User-Key", "").strip()
|
|
109
|
+
return raw or None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
async def _resolve_auth(request: Request) -> AuthContext:
|
|
113
|
+
"""Resolve the caller's tier or raise 401 if no key was sent.
|
|
114
|
+
|
|
115
|
+
Delegates classification to the cached whoami resolver. Returns
|
|
116
|
+
``AuthContext(tier=PUBLIC, ...)`` (rather than raising) when a key was
|
|
117
|
+
sent but whoami didn't recognise it — the per-route dep decides whether
|
|
118
|
+
to 403.
|
|
119
|
+
"""
|
|
120
|
+
key = _extract_key(request)
|
|
121
|
+
path = str(request.url.path)
|
|
122
|
+
client_ip = request.client.host if request.client else "unknown"
|
|
123
|
+
|
|
124
|
+
if not key:
|
|
125
|
+
log.warning("auth_missing_key", path=path, client_ip=client_ip)
|
|
126
|
+
raise AuthenticationError("X-User-Key header is required")
|
|
127
|
+
|
|
128
|
+
masked = _mask(key)
|
|
129
|
+
# The resolver returns one of the tiers in `whoami._VALID_TIERS` (it clamps
|
|
130
|
+
# anything unrecognised to "public"). This raises ValueError if that list and
|
|
131
|
+
# `Tier` ever drift apart again — keep them in step; the test asserts it.
|
|
132
|
+
identity = resolve_identity_from_key(key)
|
|
133
|
+
tier = Tier(identity.tier)
|
|
134
|
+
|
|
135
|
+
if tier is Tier.PUBLIC:
|
|
136
|
+
log.warning("auth_unknown_key", path=path, client_ip=client_ip, masked_key=masked)
|
|
137
|
+
else:
|
|
138
|
+
log.debug("auth_resolved", path=path, tier=tier.value, masked_key=masked)
|
|
139
|
+
|
|
140
|
+
return AuthContext(
|
|
141
|
+
tier=tier, masked_key=masked, user_id=identity.user_id, member=identity.member
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def require_admin(auth: AuthContext = Depends(_resolve_auth)) -> AuthContext:
|
|
146
|
+
"""Admin-only. Used for: operator-only routes, anything that touches
|
|
147
|
+
credentials or executes destructive flows."""
|
|
148
|
+
if not tier_at_least(auth.tier, Tier.ADMIN):
|
|
149
|
+
raise AuthorizationError(f"admin tier required (got {auth.tier.value})")
|
|
150
|
+
return auth
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def require_service(
|
|
154
|
+
request: Request, auth: AuthContext = Depends(_resolve_auth)
|
|
155
|
+
) -> AuthContext:
|
|
156
|
+
"""Minimum tier service, and on the ``scraper-service`` list. Default for
|
|
157
|
+
routes that mutate state / call external APIs / consume sibling-service
|
|
158
|
+
resources."""
|
|
159
|
+
if not tier_at_least(auth.tier, Tier.SERVICE):
|
|
160
|
+
raise AuthorizationError(f"service tier required (got {auth.tier.value})")
|
|
161
|
+
if not auth.member:
|
|
162
|
+
# The user id names who to add (`access-list add scraper-service <name>`)
|
|
163
|
+
# if this caller is one the scraper should serve.
|
|
164
|
+
log.warning(
|
|
165
|
+
"auth_not_listed",
|
|
166
|
+
path=str(request.url.path),
|
|
167
|
+
tier=auth.tier.value,
|
|
168
|
+
user_id=auth.user_id,
|
|
169
|
+
)
|
|
170
|
+
raise AuthorizationError("not on the scraper-service access list")
|
|
171
|
+
return auth
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def require_friend(auth: AuthContext = Depends(_resolve_auth)) -> AuthContext:
|
|
175
|
+
"""Minimum tier: friend. Use for dashboard / observability surfaces and
|
|
176
|
+
contributor-friendly trigger endpoints — anything a trusted human
|
|
177
|
+
should be able to hit. Service + admin pass too (admin always passes
|
|
178
|
+
everything by virtue of being the top tier)."""
|
|
179
|
+
if not tier_at_least(auth.tier, Tier.FRIEND):
|
|
180
|
+
raise AuthorizationError(f"friend tier required (got {auth.tier.value})")
|
|
181
|
+
return auth
|