hadoku-scrape-sdk 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. hadoku_scrape_sdk-0.1.3/LICENSE +15 -0
  2. hadoku_scrape_sdk-0.1.3/PKG-INFO +103 -0
  3. hadoku_scrape_sdk-0.1.3/README.md +76 -0
  4. hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/PKG-INFO +103 -0
  5. hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/SOURCES.txt +60 -0
  6. hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/dependency_links.txt +1 -0
  7. hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/requires.txt +18 -0
  8. hadoku_scrape_sdk-0.1.3/hadoku_scrape_sdk.egg-info/top_level.txt +1 -0
  9. hadoku_scrape_sdk-0.1.3/hadoku_sdk/__init__.py +1 -0
  10. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/__init__.py +1 -0
  11. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/__init__.py +1 -0
  12. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/auth.py +181 -0
  13. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/errors.py +296 -0
  14. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/fire_and_forget.py +159 -0
  15. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/lock_release.py +30 -0
  16. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/middleware/whoami.py +205 -0
  17. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/models/__init__.py +1 -0
  18. hadoku_scrape_sdk-0.1.3/hadoku_sdk/api/models/responses.py +25 -0
  19. hadoku_scrape_sdk-0.1.3/hadoku_sdk/auth/__init__.py +1 -0
  20. hadoku_scrape_sdk-0.1.3/hadoku_sdk/auth/cookies.py +317 -0
  21. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/__init__.py +23 -0
  22. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/cdpfree.py +560 -0
  23. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/linkedin.py +25 -0
  24. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/manager.py +1002 -0
  25. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/preflight.py +556 -0
  26. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/session.py +405 -0
  27. hadoku_scrape_sdk-0.1.3/hadoku_sdk/browser/stealth.py +254 -0
  28. hadoku_scrape_sdk-0.1.3/hadoku_sdk/config.py +45 -0
  29. hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/__init__.py +1 -0
  30. hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/engine.py +378 -0
  31. hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/execution_log.py +726 -0
  32. hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/migrations/001_execution_log.sql +28 -0
  33. hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/migrations/002_run_pacing.sql +20 -0
  34. hadoku_scrape_sdk-0.1.3/hadoku_sdk/db/migrations/003_execution_log_owner.sql +15 -0
  35. hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/__init__.py +8 -0
  36. hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/aggregator.py +107 -0
  37. hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/discord_notify.py +112 -0
  38. hadoku_scrape_sdk-0.1.3/hadoku_sdk/observability/logging.py +95 -0
  39. hadoku_scrape_sdk-0.1.3/hadoku_sdk/paths.py +28 -0
  40. hadoku_scrape_sdk-0.1.3/hadoku_sdk/plugins/__init__.py +290 -0
  41. hadoku_scrape_sdk-0.1.3/hadoku_sdk/plugins/__main__.py +28 -0
  42. hadoku_scrape_sdk-0.1.3/hadoku_sdk/plugins/manifest.py +115 -0
  43. hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/__init__.py +12 -0
  44. hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/base.py +125 -0
  45. hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/client.py +284 -0
  46. hadoku_scrape_sdk-0.1.3/hadoku_sdk/proxy/escalation.py +270 -0
  47. hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/__init__.py +36 -0
  48. hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/events.py +170 -0
  49. hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/gate.py +50 -0
  50. hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/pacing.py +227 -0
  51. hadoku_scrape_sdk-0.1.3/hadoku_sdk/runtime/reaper.py +98 -0
  52. hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/__init__.py +1 -0
  53. hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/base.py +274 -0
  54. hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/__init__.py +1 -0
  55. hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/cloudflare_errors.py +89 -0
  56. hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/cloudflare_kv.py +245 -0
  57. hadoku_scrape_sdk-0.1.3/hadoku_sdk/scrapers/utils/config.py +40 -0
  58. hadoku_scrape_sdk-0.1.3/hadoku_sdk/testing/__init__.py +1 -0
  59. hadoku_scrape_sdk-0.1.3/hadoku_sdk/testing/pytest_plugin.py +265 -0
  60. hadoku_scrape_sdk-0.1.3/hadoku_sdk/testing/unit.py +40 -0
  61. hadoku_scrape_sdk-0.1.3/pyproject.toml +131 -0
  62. hadoku_scrape_sdk-0.1.3/setup.cfg +4 -0
@@ -0,0 +1,15 @@
1
+ ISC License
2
+
3
+ Copyright (c) 2026 WolffM
4
+
5
+ Permission to use, copy, modify, and/or distribute this software for any
6
+ purpose with or without fee is hereby granted, provided that the above
7
+ copyright notice and this permission notice appear in all copies.
8
+
9
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12
+ ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
@@ -0,0 +1,103 @@
1
+ Metadata-Version: 2.4
2
+ Name: hadoku-scrape-sdk
3
+ Version: 0.1.3
4
+ Summary: The runtime hadoku-scrape plugins build on: plugin contract, browser stack, run log, events, test floor
5
+ License-Expression: ISC
6
+ Requires-Python: >=3.11
7
+ Description-Content-Type: text/markdown
8
+ License-File: LICENSE
9
+ Requires-Dist: browser-cookie3<1.0,>=0.19.0
10
+ Requires-Dist: fastapi<1.0,>=0.109.0
11
+ Requires-Dist: hadoku-execlog>=0.1.0
12
+ Requires-Dist: httpx[socks]<1.0,>=0.27.0
13
+ Requires-Dist: patchright<2.0,>=1.49.0
14
+ Requires-Dist: playwright<2.0,>=1.62.0
15
+ Requires-Dist: pydantic<3.0,>=2.0
16
+ Requires-Dist: pyvirtualdisplay<4.0,>=3.0
17
+ Requires-Dist: structlog<25.0,>=24.0.0
18
+ Requires-Dist: tenacity<9.0,>=8.2.0
19
+ Provides-Extra: testing
20
+ Requires-Dist: pytest==8.4.2; extra == "testing"
21
+ Requires-Dist: pytest-asyncio<2,>=1.2; extra == "testing"
22
+ Requires-Dist: pytest-timeout<3.0,>=2.2.0; extra == "testing"
23
+ Requires-Dist: respx<1.0,>=0.21.0; extra == "testing"
24
+ Requires-Dist: httpx2<3.0,>=2.9; extra == "testing"
25
+ Requires-Dist: typer<1.0,>=0.15.0; extra == "testing"
26
+ Dynamic: license-file
27
+
28
+ # hadoku-scrape-sdk
29
+
30
+ The runtime that hadoku-scrape plugins are built on. A plugin is a scraper that
31
+ lives in its own repo and runs inside the hadoku-scrape service. This package is
32
+ everything such a plugin imports, and nothing of the service around it.
33
+
34
+ ```sh
35
+ pip install hadoku-scrape-sdk # runtime
36
+ pip install 'hadoku-scrape-sdk[testing]' # + the pinned test stack and test floor
37
+ ```
38
+
39
+ ## What's in it
40
+
41
+ | Module | What it gives a plugin |
42
+ |---|---|
43
+ | `hadoku_sdk.plugins` | `ScraperPlugin`, `UrlScraper`: the contract. `python -m hadoku_sdk.plugins` checks every enabled plugin loads |
44
+ | `hadoku_sdk.db.engine`, `.db.execution_log` | The SQLite engine (`init_db`, `get_connection`) and `execution_run`, the run log every scrape writes through |
45
+ | `hadoku_sdk.runtime` | `Severity` and events (`ctx.event(...)`), run pacing, the run gate |
46
+ | `hadoku_sdk.browser` | Browser manager (Patchright, persistent profiles, stealth), sessions, preflight checks |
47
+ | `hadoku_sdk.proxy` | HTTP client with proxy escalation |
48
+ | `hadoku_sdk.auth.cookies` | Browser-cookie import |
49
+ | `hadoku_sdk.api.middleware` | Auth dependencies (`require_service`, ...), error types, `spawn_tracked` for fire-and-forget routes |
50
+ | `hadoku_sdk.scrapers.base` | `BaseScraper`, `ScraperResult` for URL-dispatch scrapers |
51
+ | `hadoku_sdk.observability.logging` | `get_logger` (structlog) |
52
+ | `hadoku_sdk.testing` | The autouse test floor: an isolated DB per test, no real browsers |
53
+
54
+ ## A plugin
55
+
56
+ ```python
57
+ # my_plugins/thing/plugin.py
58
+ from pathlib import Path
59
+ from hadoku_sdk.plugins import ScraperPlugin
60
+
61
+ PLUGIN = ScraperPlugin(
62
+ name="thing",
63
+ routers=[router], # FastAPI, prefix /api/v1/<yours>
64
+ cli={"thing": thing_app}, # Typer, mounted as `hs thing`
65
+ migrations=Path(__file__).parent / "migrations", # NNN_*.sql
66
+ on_startup=[seed],
67
+ )
68
+ ```
69
+
70
+ ```toml
71
+ # pyproject.toml
72
+ [project.entry-points."hadoku_scrape.plugins"]
73
+ thing = "my_plugins.thing.plugin:PLUGIN"
74
+ ```
75
+
76
+ The host's `plugins.toml` decides which installed plugins actually load.
77
+
78
+ ## Tests
79
+
80
+ ```python
81
+ # tests/conftest.py
82
+ import os
83
+ from pathlib import Path
84
+ os.environ["HADOKU_PLUGINS_MANIFEST"] = str(Path(__file__).parent / "resources" / "plugins.toml")
85
+ pytest_plugins = ["hadoku_sdk.testing.pytest_plugin"]
86
+ ```
87
+
88
+ ```python
89
+ # tests/unit/conftest.py
90
+ from hadoku_sdk.testing.unit import _instant_sleep, _silence_discord_notify # noqa: F401
91
+ ```
92
+
93
+ ## Rules a plugin follows
94
+
95
+ - Notifications go through `ctx.event(name, payload, severity=...)` on the run
96
+ context, never a direct Discord call. The host routes them.
97
+ - Timestamps are UTC. Never `datetime.utcnow()`.
98
+ - Never log credentials, cookies or API keys.
99
+ - No lazy imports inside functions that run on background threads.
100
+ - Drive browsers through `hadoku_sdk.browser.manager`. Never hardcode locale or
101
+ timezone.
102
+
103
+ License: ISC.
@@ -0,0 +1,76 @@
1
+ # hadoku-scrape-sdk
2
+
3
+ The runtime that hadoku-scrape plugins are built on. A plugin is a scraper that
4
+ lives in its own repo and runs inside the hadoku-scrape service. This package is
5
+ everything such a plugin imports, and nothing of the service around it.
6
+
7
+ ```sh
8
+ pip install hadoku-scrape-sdk # runtime
9
+ pip install 'hadoku-scrape-sdk[testing]' # + the pinned test stack and test floor
10
+ ```
11
+
12
+ ## What's in it
13
+
14
+ | Module | What it gives a plugin |
15
+ |---|---|
16
+ | `hadoku_sdk.plugins` | `ScraperPlugin`, `UrlScraper`: the contract. `python -m hadoku_sdk.plugins` checks every enabled plugin loads |
17
+ | `hadoku_sdk.db.engine`, `.db.execution_log` | The SQLite engine (`init_db`, `get_connection`) and `execution_run`, the run log every scrape writes through |
18
+ | `hadoku_sdk.runtime` | `Severity` and events (`ctx.event(...)`), run pacing, the run gate |
19
+ | `hadoku_sdk.browser` | Browser manager (Patchright, persistent profiles, stealth), sessions, preflight checks |
20
+ | `hadoku_sdk.proxy` | HTTP client with proxy escalation |
21
+ | `hadoku_sdk.auth.cookies` | Browser-cookie import |
22
+ | `hadoku_sdk.api.middleware` | Auth dependencies (`require_service`, ...), error types, `spawn_tracked` for fire-and-forget routes |
23
+ | `hadoku_sdk.scrapers.base` | `BaseScraper`, `ScraperResult` for URL-dispatch scrapers |
24
+ | `hadoku_sdk.observability.logging` | `get_logger` (structlog) |
25
+ | `hadoku_sdk.testing` | The autouse test floor: an isolated DB per test, no real browsers |
26
+
27
+ ## A plugin
28
+
29
+ ```python
30
+ # my_plugins/thing/plugin.py
31
+ from pathlib import Path
32
+ from hadoku_sdk.plugins import ScraperPlugin
33
+
34
+ PLUGIN = ScraperPlugin(
35
+ name="thing",
36
+ routers=[router], # FastAPI, prefix /api/v1/<yours>
37
+ cli={"thing": thing_app}, # Typer, mounted as `hs thing`
38
+ migrations=Path(__file__).parent / "migrations", # NNN_*.sql
39
+ on_startup=[seed],
40
+ )
41
+ ```
42
+
43
+ ```toml
44
+ # pyproject.toml
45
+ [project.entry-points."hadoku_scrape.plugins"]
46
+ thing = "my_plugins.thing.plugin:PLUGIN"
47
+ ```
48
+
49
+ The host's `plugins.toml` decides which installed plugins actually load.
50
+
51
+ ## Tests
52
+
53
+ ```python
54
+ # tests/conftest.py
55
+ import os
56
+ from pathlib import Path
57
+ os.environ["HADOKU_PLUGINS_MANIFEST"] = str(Path(__file__).parent / "resources" / "plugins.toml")
58
+ pytest_plugins = ["hadoku_sdk.testing.pytest_plugin"]
59
+ ```
60
+
61
+ ```python
62
+ # tests/unit/conftest.py
63
+ from hadoku_sdk.testing.unit import _instant_sleep, _silence_discord_notify # noqa: F401
64
+ ```
65
+
66
+ ## Rules a plugin follows
67
+
68
+ - Notifications go through `ctx.event(name, payload, severity=...)` on the run
69
+ context, never a direct Discord call. The host routes them.
70
+ - Timestamps are UTC. Never `datetime.utcnow()`.
71
+ - Never log credentials, cookies or API keys.
72
+ - No lazy imports inside functions that run on background threads.
73
+ - Drive browsers through `hadoku_sdk.browser.manager`. Never hardcode locale or
74
+ timezone.
75
+
76
+ License: ISC.
@@ -0,0 +1,103 @@
1
+ Metadata-Version: 2.4
2
+ Name: hadoku-scrape-sdk
3
+ Version: 0.1.3
4
+ Summary: The runtime hadoku-scrape plugins build on: plugin contract, browser stack, run log, events, test floor
5
+ License-Expression: ISC
6
+ Requires-Python: >=3.11
7
+ Description-Content-Type: text/markdown
8
+ License-File: LICENSE
9
+ Requires-Dist: browser-cookie3<1.0,>=0.19.0
10
+ Requires-Dist: fastapi<1.0,>=0.109.0
11
+ Requires-Dist: hadoku-execlog>=0.1.0
12
+ Requires-Dist: httpx[socks]<1.0,>=0.27.0
13
+ Requires-Dist: patchright<2.0,>=1.49.0
14
+ Requires-Dist: playwright<2.0,>=1.62.0
15
+ Requires-Dist: pydantic<3.0,>=2.0
16
+ Requires-Dist: pyvirtualdisplay<4.0,>=3.0
17
+ Requires-Dist: structlog<25.0,>=24.0.0
18
+ Requires-Dist: tenacity<9.0,>=8.2.0
19
+ Provides-Extra: testing
20
+ Requires-Dist: pytest==8.4.2; extra == "testing"
21
+ Requires-Dist: pytest-asyncio<2,>=1.2; extra == "testing"
22
+ Requires-Dist: pytest-timeout<3.0,>=2.2.0; extra == "testing"
23
+ Requires-Dist: respx<1.0,>=0.21.0; extra == "testing"
24
+ Requires-Dist: httpx2<3.0,>=2.9; extra == "testing"
25
+ Requires-Dist: typer<1.0,>=0.15.0; extra == "testing"
26
+ Dynamic: license-file
27
+
28
+ # hadoku-scrape-sdk
29
+
30
+ The runtime that hadoku-scrape plugins are built on. A plugin is a scraper that
31
+ lives in its own repo and runs inside the hadoku-scrape service. This package is
32
+ everything such a plugin imports, and nothing of the service around it.
33
+
34
+ ```sh
35
+ pip install hadoku-scrape-sdk # runtime
36
+ pip install 'hadoku-scrape-sdk[testing]' # + the pinned test stack and test floor
37
+ ```
38
+
39
+ ## What's in it
40
+
41
+ | Module | What it gives a plugin |
42
+ |---|---|
43
+ | `hadoku_sdk.plugins` | `ScraperPlugin`, `UrlScraper`: the contract. `python -m hadoku_sdk.plugins` checks every enabled plugin loads |
44
+ | `hadoku_sdk.db.engine`, `.db.execution_log` | The SQLite engine (`init_db`, `get_connection`) and `execution_run`, the run log every scrape writes through |
45
+ | `hadoku_sdk.runtime` | `Severity` and events (`ctx.event(...)`), run pacing, the run gate |
46
+ | `hadoku_sdk.browser` | Browser manager (Patchright, persistent profiles, stealth), sessions, preflight checks |
47
+ | `hadoku_sdk.proxy` | HTTP client with proxy escalation |
48
+ | `hadoku_sdk.auth.cookies` | Browser-cookie import |
49
+ | `hadoku_sdk.api.middleware` | Auth dependencies (`require_service`, ...), error types, `spawn_tracked` for fire-and-forget routes |
50
+ | `hadoku_sdk.scrapers.base` | `BaseScraper`, `ScraperResult` for URL-dispatch scrapers |
51
+ | `hadoku_sdk.observability.logging` | `get_logger` (structlog) |
52
+ | `hadoku_sdk.testing` | The autouse test floor: an isolated DB per test, no real browsers |
53
+
54
+ ## A plugin
55
+
56
+ ```python
57
+ # my_plugins/thing/plugin.py
58
+ from pathlib import Path
59
+ from hadoku_sdk.plugins import ScraperPlugin
60
+
61
+ PLUGIN = ScraperPlugin(
62
+ name="thing",
63
+ routers=[router], # FastAPI, prefix /api/v1/<yours>
64
+ cli={"thing": thing_app}, # Typer, mounted as `hs thing`
65
+ migrations=Path(__file__).parent / "migrations", # NNN_*.sql
66
+ on_startup=[seed],
67
+ )
68
+ ```
69
+
70
+ ```toml
71
+ # pyproject.toml
72
+ [project.entry-points."hadoku_scrape.plugins"]
73
+ thing = "my_plugins.thing.plugin:PLUGIN"
74
+ ```
75
+
76
+ The host's `plugins.toml` decides which installed plugins actually load.
77
+
78
+ ## Tests
79
+
80
+ ```python
81
+ # tests/conftest.py
82
+ import os
83
+ from pathlib import Path
84
+ os.environ["HADOKU_PLUGINS_MANIFEST"] = str(Path(__file__).parent / "resources" / "plugins.toml")
85
+ pytest_plugins = ["hadoku_sdk.testing.pytest_plugin"]
86
+ ```
87
+
88
+ ```python
89
+ # tests/unit/conftest.py
90
+ from hadoku_sdk.testing.unit import _instant_sleep, _silence_discord_notify # noqa: F401
91
+ ```
92
+
93
+ ## Rules a plugin follows
94
+
95
+ - Notifications go through `ctx.event(name, payload, severity=...)` on the run
96
+ context, never a direct Discord call. The host routes them.
97
+ - Timestamps are UTC. Never `datetime.utcnow()`.
98
+ - Never log credentials, cookies or API keys.
99
+ - No lazy imports inside functions that run on background threads.
100
+ - Drive browsers through `hadoku_sdk.browser.manager`. Never hardcode locale or
101
+ timezone.
102
+
103
+ License: ISC.
@@ -0,0 +1,60 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ hadoku_scrape_sdk.egg-info/PKG-INFO
5
+ hadoku_scrape_sdk.egg-info/SOURCES.txt
6
+ hadoku_scrape_sdk.egg-info/dependency_links.txt
7
+ hadoku_scrape_sdk.egg-info/requires.txt
8
+ hadoku_scrape_sdk.egg-info/top_level.txt
9
+ hadoku_sdk/__init__.py
10
+ hadoku_sdk/config.py
11
+ hadoku_sdk/paths.py
12
+ hadoku_sdk/api/__init__.py
13
+ hadoku_sdk/api/middleware/__init__.py
14
+ hadoku_sdk/api/middleware/auth.py
15
+ hadoku_sdk/api/middleware/errors.py
16
+ hadoku_sdk/api/middleware/fire_and_forget.py
17
+ hadoku_sdk/api/middleware/lock_release.py
18
+ hadoku_sdk/api/middleware/whoami.py
19
+ hadoku_sdk/api/models/__init__.py
20
+ hadoku_sdk/api/models/responses.py
21
+ hadoku_sdk/auth/__init__.py
22
+ hadoku_sdk/auth/cookies.py
23
+ hadoku_sdk/browser/__init__.py
24
+ hadoku_sdk/browser/cdpfree.py
25
+ hadoku_sdk/browser/linkedin.py
26
+ hadoku_sdk/browser/manager.py
27
+ hadoku_sdk/browser/preflight.py
28
+ hadoku_sdk/browser/session.py
29
+ hadoku_sdk/browser/stealth.py
30
+ hadoku_sdk/db/__init__.py
31
+ hadoku_sdk/db/engine.py
32
+ hadoku_sdk/db/execution_log.py
33
+ hadoku_sdk/db/migrations/001_execution_log.sql
34
+ hadoku_sdk/db/migrations/002_run_pacing.sql
35
+ hadoku_sdk/db/migrations/003_execution_log_owner.sql
36
+ hadoku_sdk/observability/__init__.py
37
+ hadoku_sdk/observability/aggregator.py
38
+ hadoku_sdk/observability/discord_notify.py
39
+ hadoku_sdk/observability/logging.py
40
+ hadoku_sdk/plugins/__init__.py
41
+ hadoku_sdk/plugins/__main__.py
42
+ hadoku_sdk/plugins/manifest.py
43
+ hadoku_sdk/proxy/__init__.py
44
+ hadoku_sdk/proxy/base.py
45
+ hadoku_sdk/proxy/client.py
46
+ hadoku_sdk/proxy/escalation.py
47
+ hadoku_sdk/runtime/__init__.py
48
+ hadoku_sdk/runtime/events.py
49
+ hadoku_sdk/runtime/gate.py
50
+ hadoku_sdk/runtime/pacing.py
51
+ hadoku_sdk/runtime/reaper.py
52
+ hadoku_sdk/scrapers/__init__.py
53
+ hadoku_sdk/scrapers/base.py
54
+ hadoku_sdk/scrapers/utils/__init__.py
55
+ hadoku_sdk/scrapers/utils/cloudflare_errors.py
56
+ hadoku_sdk/scrapers/utils/cloudflare_kv.py
57
+ hadoku_sdk/scrapers/utils/config.py
58
+ hadoku_sdk/testing/__init__.py
59
+ hadoku_sdk/testing/pytest_plugin.py
60
+ hadoku_sdk/testing/unit.py
@@ -0,0 +1,18 @@
1
+ browser-cookie3<1.0,>=0.19.0
2
+ fastapi<1.0,>=0.109.0
3
+ hadoku-execlog>=0.1.0
4
+ httpx[socks]<1.0,>=0.27.0
5
+ patchright<2.0,>=1.49.0
6
+ playwright<2.0,>=1.62.0
7
+ pydantic<3.0,>=2.0
8
+ pyvirtualdisplay<4.0,>=3.0
9
+ structlog<25.0,>=24.0.0
10
+ tenacity<9.0,>=8.2.0
11
+
12
+ [testing]
13
+ pytest==8.4.2
14
+ pytest-asyncio<2,>=1.2
15
+ pytest-timeout<3.0,>=2.2.0
16
+ respx<1.0,>=0.21.0
17
+ httpx2<3.0,>=2.9
18
+ typer<1.0,>=0.15.0
@@ -0,0 +1 @@
1
+ """hadoku-scrape SDK: what a scraper plugin builds on, without the service around it."""
@@ -0,0 +1 @@
1
+ """Part of hadoku-scrape-sdk."""
@@ -0,0 +1 @@
1
+ """Part of hadoku-scrape-sdk."""
@@ -0,0 +1,181 @@
1
+ """Tier-aware API authentication.
2
+
3
+ The edge in front of the service resolves the caller's session/key and
4
+ injects ``X-User-Key`` on the proxied request. The backend
5
+ holds **no key arrays of its own** — it delegates tier resolution to the
6
+ platform's authoritative ``/session/whoami`` endpoint (see
7
+ :mod:`hadoku_sdk.api.middleware.whoami`), cached per key. This works for
8
+ both edge-tunnelled callers and the direct service-to-service callers
9
+ that bypass the edge.
10
+
11
+ Tiers:
12
+ - ``admin`` operator key, full access
13
+ - ``service`` internal sibling-service key — full access EXCEPT routes explicitly
14
+ admin-only, and only for a key on the ``scraper-service``
15
+ access list
16
+ - ``friend`` trusted human, read-only observability access
17
+ - ``public`` no key / unrecognised key
18
+
19
+ Per-route deps name the *minimum* tier accepted; admin is implicit.
20
+
21
+ - ``require_admin`` admin only
22
+ - ``require_service`` service or higher AND on ``scraper-service`` (default
23
+ for routes that mutate state / consume resources /
24
+ call external APIs). Service tier is every service
25
+ key on the platform, and people hold them too; the list
26
+ names the callers that drive the scraper. Admin is
27
+ not exempt: the operator is on the list.
28
+ - ``require_friend`` friend or higher (dashboard / observability /
29
+ contributor-friendly trigger endpoints — anything
30
+ a trusted human should be able to hit)
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ from dataclasses import dataclass
36
+ from enum import Enum
37
+
38
+ from fastapi import Depends, Request
39
+
40
+ from hadoku_sdk.api.middleware.errors import AuthenticationError, AuthorizationError
41
+ from hadoku_sdk.api.middleware.whoami import resolve_identity_from_key
42
+ from hadoku_sdk.observability.logging import get_logger
43
+
44
+ log = get_logger("api.auth")
45
+
46
+
47
+ class Tier(str, Enum):
48
+ """The tier ladder. Mirrors `_VALID_TIERS` in `whoami.py`: if a tier exists
49
+ there and not here, `_resolve_auth` raises `ValueError` on it, which is a 500
50
+ rather than a 403. Edge tier names beyond these are mapped onto them with
51
+ `whoami.alias_tier`.
52
+ """
53
+
54
+ ADMIN = "admin"
55
+ MEMBER = "member"
56
+ SERVICE = "service"
57
+ FRIEND = "friend"
58
+ PUBLIC = "public"
59
+
60
+
61
+ # LOW to HIGH. Gates compare RANK, never identity or membership: `tier is Tier.ADMIN`
62
+ # and `tier not in (Tier.ADMIN, Tier.SERVICE)` both lock out any tier that sits ABOVE
63
+ # the one being named, which is the opposite of what a minimum-tier gate means.
64
+ _RANK: dict[Tier, int] = {
65
+ Tier.PUBLIC: 0,
66
+ Tier.FRIEND: 1,
67
+ Tier.SERVICE: 2,
68
+ Tier.MEMBER: 3,
69
+ Tier.ADMIN: 4,
70
+ }
71
+
72
+
73
+ def tier_at_least(tier: Tier, minimum: Tier) -> bool:
74
+ """True when `tier` sits at or above `minimum` on the ladder."""
75
+ return _RANK.get(tier, -1) >= _RANK[minimum]
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class AuthContext:
80
+ """Resolved caller identity. Yielded by the require_* deps so route
81
+ handlers can stamp audit fields without ever touching the raw key."""
82
+
83
+ tier: Tier
84
+ masked_key: str
85
+ #: The platform's stable per-user UUID, straight from whoami — which the
86
+ #: resolver has to call anyway to get the tier. Durable across sessions and
87
+ #: devices, so it is the right key for anything stored per person. None
88
+ #: when whoami did not supply one (anonymous, transient failure, or a test
89
+ #: override), which callers must handle rather than assume.
90
+ user_id: str | None = None
91
+ #: On the ``scraper-service`` access list (whoami answers it per key).
92
+ member: bool = False
93
+
94
+ def __str__(self) -> str:
95
+ # The userId, not the masked key: this is what lands in an f-string,
96
+ # and from there in whatever the f-string is written to.
97
+ return f"{self.tier.value}:{self.user_id or 'no-user'}"
98
+
99
+
100
+ def _mask(key: str) -> str:
101
+ return f"{key[:4]}…{key[-2:]}" if len(key) > 8 else "****"
102
+
103
+
104
+ def _extract_key(request: Request) -> str | None:
105
+ """Read the caller key from ``X-User-Key``. Bearer tokens are no longer
106
+ accepted at the backend — the edge translates session/cookie/Bearer
107
+ forms into ``X-User-Key`` before proxying."""
108
+ raw = request.headers.get("X-User-Key", "").strip()
109
+ return raw or None
110
+
111
+
112
+ async def _resolve_auth(request: Request) -> AuthContext:
113
+ """Resolve the caller's tier or raise 401 if no key was sent.
114
+
115
+ Delegates classification to the cached whoami resolver. Returns
116
+ ``AuthContext(tier=PUBLIC, ...)`` (rather than raising) when a key was
117
+ sent but whoami didn't recognise it — the per-route dep decides whether
118
+ to 403.
119
+ """
120
+ key = _extract_key(request)
121
+ path = str(request.url.path)
122
+ client_ip = request.client.host if request.client else "unknown"
123
+
124
+ if not key:
125
+ log.warning("auth_missing_key", path=path, client_ip=client_ip)
126
+ raise AuthenticationError("X-User-Key header is required")
127
+
128
+ masked = _mask(key)
129
+ # The resolver returns one of the tiers in `whoami._VALID_TIERS` (it clamps
130
+ # anything unrecognised to "public"). This raises ValueError if that list and
131
+ # `Tier` ever drift apart again — keep them in step; the test asserts it.
132
+ identity = resolve_identity_from_key(key)
133
+ tier = Tier(identity.tier)
134
+
135
+ if tier is Tier.PUBLIC:
136
+ log.warning("auth_unknown_key", path=path, client_ip=client_ip, masked_key=masked)
137
+ else:
138
+ log.debug("auth_resolved", path=path, tier=tier.value, masked_key=masked)
139
+
140
+ return AuthContext(
141
+ tier=tier, masked_key=masked, user_id=identity.user_id, member=identity.member
142
+ )
143
+
144
+
145
+ def require_admin(auth: AuthContext = Depends(_resolve_auth)) -> AuthContext:
146
+ """Admin-only. Used for: operator-only routes, anything that touches
147
+ credentials or executes destructive flows."""
148
+ if not tier_at_least(auth.tier, Tier.ADMIN):
149
+ raise AuthorizationError(f"admin tier required (got {auth.tier.value})")
150
+ return auth
151
+
152
+
153
+ def require_service(
154
+ request: Request, auth: AuthContext = Depends(_resolve_auth)
155
+ ) -> AuthContext:
156
+ """Minimum tier service, and on the ``scraper-service`` list. Default for
157
+ routes that mutate state / call external APIs / consume sibling-service
158
+ resources."""
159
+ if not tier_at_least(auth.tier, Tier.SERVICE):
160
+ raise AuthorizationError(f"service tier required (got {auth.tier.value})")
161
+ if not auth.member:
162
+ # The user id names who to add (`access-list add scraper-service <name>`)
163
+ # if this caller is one the scraper should serve.
164
+ log.warning(
165
+ "auth_not_listed",
166
+ path=str(request.url.path),
167
+ tier=auth.tier.value,
168
+ user_id=auth.user_id,
169
+ )
170
+ raise AuthorizationError("not on the scraper-service access list")
171
+ return auth
172
+
173
+
174
+ def require_friend(auth: AuthContext = Depends(_resolve_auth)) -> AuthContext:
175
+ """Minimum tier: friend. Use for dashboard / observability surfaces and
176
+ contributor-friendly trigger endpoints — anything a trusted human
177
+ should be able to hit. Service + admin pass too (admin always passes
178
+ everything by virtue of being the top tier)."""
179
+ if not tier_at_least(auth.tier, Tier.FRIEND):
180
+ raise AuthorizationError(f"friend tier required (got {auth.tier.value})")
181
+ return auth