archivebox-api 2.5.1__tar.gz → 2.5.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/AGENTS.md +14 -2
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/PKG-INFO +5 -6
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/README.md +1 -1
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/api/api_client_auth.py +3 -3
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/api/api_client_base.py +11 -3
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/api/api_client_cli.py +6 -6
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/api/api_client_core.py +8 -8
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/connector_manifest.yml +1 -1
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/connectors/tool_schema_fingerprints.json +1 -1
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/kg_ingest.py +6 -6
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/mcp_server.py +6 -22
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/web_source.py +35 -31
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/web_source_contract.py +15 -3
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api.egg-info/PKG-INFO +5 -6
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api.egg-info/SOURCES.txt +1 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api.egg-info/entry_points.txt +3 -0
- archivebox_api-2.5.3/archivebox_api.egg-info/requires.txt +13 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/connector_manifest.yml +1 -1
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/pyproject.toml +8 -4
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/release-readiness.json +770 -1407
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_archivebox_api_brute_force_coverage.py +1 -3
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_archivebox_coverage.py +1 -1
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_archivebox_coverage_extra.py +15 -42
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_basic.py +1 -1
- archivebox_api-2.5.3/tests/test_sdk_transport.py +74 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_web_source.py +21 -0
- archivebox_api-2.5.1/archivebox_api.egg-info/requires.txt +0 -14
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/.pre-commit-config.yaml +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/CHANGELOG.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/CLAUDE.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/LICENSE +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/MANIFEST.in +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/__init__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/__main__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/api/__init__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/api_client.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/auth.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/connectors/__init__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/connectors/mcp_source_presets.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/mcp_config.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/__init__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/archivebox.ttl +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/certification.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/fixtures/records.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/mappings/source.yaml +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/migrations/manifest.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/ontology/shapes/connector.shacl.ttl +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/prompts/__init__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/prompts/web_archivist_specialist.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/__init__.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-api-operations/SKILL.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-api-operations/agents/openai.yaml +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-api-operations/references/catalog.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-kg-ingestion/SKILL.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-kg-ingestion/WORKFLOW.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-snapshot-search/SKILL.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-snapshot-search/WORKFLOW.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-web-archiving/SKILL.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api/skills/archivebox-web-archiving/WORKFLOW.md +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api.egg-info/dependency_links.txt +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/archivebox_api.egg-info/top_level.txt +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/mkdocs.yml +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/opencode.json +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/requirements.txt +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/scripts/security_sanitizer.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/scripts/verify_api_integration.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/setup.cfg +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/conftest.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_action_discovery.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_conformance.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_init_dynamics.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_kg_ingest.py +0 -0
- {archivebox_api-2.5.1 → archivebox_api-2.5.3}/tests/test_kg_media.py +0 -0
|
@@ -1,7 +1,19 @@
|
|
|
1
1
|
# AGENTS.md
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
<!-- agent-contract:start -->
|
|
4
|
+
## Agent contract
|
|
5
|
+
|
|
6
|
+
Tool-neutral rules for any AI agent or human working in this repository. This block is maintained by the fleet standard; edit outside the markers.
|
|
7
|
+
|
|
8
|
+
- **Owner boundary:** this repository owns one external system's calls, mappings and MCP tools (spec prefix `CONN-ARCHIVEBOX-API-`); it depends on the connector SDK only. Feature routing across repositories: [ecosystem boundaries](https://github.com/Knuckles-Team/pipelines/blob/main/reference/ecosystem-boundaries.md).
|
|
9
|
+
- **Setup:** run `scripts/bootstrap.sh`. It is idempotent and non-interactive: it pins uv and Python, syncs the locked environment, installs the git hooks and prints the gate command.
|
|
10
|
+
- **Gates:** `uvx pre-commit run --all-files` must pass before you push. Do not bypass hooks to get green.
|
|
11
|
+
- **Commits:** one logical change per commit with a `Spec: <ID>` trailer (or `Spec: none (<reason>)`), plus a `Co-Authored-By:` trailer naming the assisting agent. Stage explicit paths only; never `git add -A` or `git add .`. Use your own configured Git identity, never a tool or bot name.
|
|
12
|
+
- **Org rules:** [organization AGENTS.md](https://github.com/Knuckles-Team/.github/blob/main/AGENTS.md).
|
|
13
|
+
<!-- agent-contract:end -->
|
|
14
|
+
|
|
15
|
+
> Canonical guidance for any AI coding agent or human; tool-specific entry files
|
|
16
|
+
> (for example `CLAUDE.md`) only import this file. Edit this file.
|
|
5
17
|
|
|
6
18
|
## Tech Stack & Architecture
|
|
7
19
|
- Language/Version: Python 3.10+
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: archivebox-api
|
|
3
|
-
Version: 2.5.
|
|
3
|
+
Version: 2.5.3
|
|
4
4
|
Summary: Pythonic ArchiveBox API Wrapper and Fast MCP Server for Agentic AI use!
|
|
5
5
|
Author-email: Project Maintainers <maintainers@users.noreply.github.com>
|
|
6
6
|
License: MIT
|
|
@@ -12,12 +12,11 @@ Classifier: Programming Language :: Python :: 3
|
|
|
12
12
|
Requires-Python: <3.15,>=3.12
|
|
13
13
|
Description-Content-Type: text/markdown
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.
|
|
16
|
-
Requires-Dist: requests>=2.34.2
|
|
15
|
+
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.11
|
|
17
16
|
Provides-Extra: mcp
|
|
18
|
-
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.
|
|
17
|
+
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.11; extra == "mcp"
|
|
19
18
|
Provides-Extra: all
|
|
20
|
-
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.
|
|
19
|
+
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.11; extra == "all"
|
|
21
20
|
Provides-Extra: test
|
|
22
21
|
Requires-Dist: pytest-xdist>=3.8.0; extra == "test"
|
|
23
22
|
Requires-Dist: pytest>=9.1.1; extra == "test"
|
|
@@ -47,7 +46,7 @@ Dynamic: license-file
|
|
|
47
46
|

|
|
48
47
|

|
|
49
48
|
|
|
50
|
-
*Version: 2.5.
|
|
49
|
+
*Version: 2.5.3*
|
|
51
50
|
|
|
52
51
|
> **Documentation** — Installation, deployment, usage across the API, CLI, MCP, and
|
|
53
52
|
> A2A agent interfaces, and guidance for provisioning the ArchiveBox platform are
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|

|
|
21
21
|

|
|
22
22
|
|
|
23
|
-
*Version: 2.5.
|
|
23
|
+
*Version: 2.5.3*
|
|
24
24
|
|
|
25
25
|
> **Documentation** — Installation, deployment, usage across the API, CLI, MCP, and
|
|
26
26
|
> A2A agent interfaces, and guidance for provisioning the ArchiveBox platform are
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import httpx
|
|
4
4
|
from agent_connector_sdk.exceptions import (
|
|
5
5
|
ParameterError,
|
|
6
6
|
)
|
|
@@ -12,7 +12,7 @@ from archivebox_api.api.api_client_base import BaseApiClient
|
|
|
12
12
|
class Api(BaseApiClient):
|
|
13
13
|
def get_api_token(
|
|
14
14
|
self, username: str | None = None, password: str | None = None
|
|
15
|
-
) ->
|
|
15
|
+
) -> httpx.Response:
|
|
16
16
|
"""
|
|
17
17
|
Generate an API token for a given username & password
|
|
18
18
|
|
|
@@ -41,7 +41,7 @@ class Api(BaseApiClient):
|
|
|
41
41
|
raise ParameterError(f"Invalid parameters: {e.errors()}") from e
|
|
42
42
|
return response
|
|
43
43
|
|
|
44
|
-
def check_api_token(self, token: str) ->
|
|
44
|
+
def check_api_token(self, token: str) -> httpx.Response:
|
|
45
45
|
"""
|
|
46
46
|
Validate an API token to make sure it's valid and non-expired
|
|
47
47
|
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from abc import ABC, abstractmethod
|
|
3
3
|
|
|
4
|
-
import
|
|
4
|
+
import httpx
|
|
5
5
|
from agent_connector_sdk.exceptions import (
|
|
6
6
|
AuthError,
|
|
7
7
|
MissingParameterError,
|
|
8
8
|
)
|
|
9
|
+
from agent_connector_sdk.http.client import create_http_client
|
|
10
|
+
from agent_connector_sdk.http.options import HttpClientOptions
|
|
9
11
|
from agent_connector_sdk.tls.profile import ResolvedTLSProfile
|
|
10
12
|
from agent_connector_sdk.tls.resolve import resolve_tls_profile
|
|
11
13
|
|
|
@@ -24,8 +26,14 @@ class BaseApiClient(ABC):
|
|
|
24
26
|
raise MissingParameterError("URL is required")
|
|
25
27
|
|
|
26
28
|
self.tls_profile = tls_profile or resolve_tls_profile("archivebox")
|
|
27
|
-
self._session = self.tls_profile.configure_requests_session(requests.Session())
|
|
28
29
|
self.url = url.rstrip("/")
|
|
30
|
+
self._session = create_http_client(
|
|
31
|
+
HttpClientOptions(
|
|
32
|
+
base_url=self.url,
|
|
33
|
+
tls=self.tls_profile,
|
|
34
|
+
allow_plaintext=self.url.startswith("http://"),
|
|
35
|
+
)
|
|
36
|
+
)
|
|
29
37
|
self.headers = {"Content-Type": "application/json"}
|
|
30
38
|
|
|
31
39
|
self._configure_auth_headers(
|
|
@@ -78,5 +86,5 @@ class BaseApiClient(ABC):
|
|
|
78
86
|
@abstractmethod
|
|
79
87
|
def get_api_token(
|
|
80
88
|
self, username: str | None = None, password: str | None = None
|
|
81
|
-
) ->
|
|
89
|
+
) -> httpx.Response:
|
|
82
90
|
raise NotImplementedError()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import httpx
|
|
4
4
|
from agent_connector_sdk.exceptions import (
|
|
5
5
|
ParameterError,
|
|
6
6
|
require_auth,
|
|
@@ -25,7 +25,7 @@ class Api(BaseApiClient):
|
|
|
25
25
|
extractors: str = "",
|
|
26
26
|
parser: str = "auto",
|
|
27
27
|
extra_data: dict | None = None,
|
|
28
|
-
) ->
|
|
28
|
+
) -> httpx.Response:
|
|
29
29
|
"""
|
|
30
30
|
Execute archivebox add command
|
|
31
31
|
|
|
@@ -86,7 +86,7 @@ class Api(BaseApiClient):
|
|
|
86
86
|
filter_patterns: list[str] | None = None,
|
|
87
87
|
extractors: str | None = "",
|
|
88
88
|
extra_data: dict | None = None,
|
|
89
|
-
) ->
|
|
89
|
+
) -> httpx.Response:
|
|
90
90
|
"""
|
|
91
91
|
Execute archivebox update command
|
|
92
92
|
|
|
@@ -148,7 +148,7 @@ class Api(BaseApiClient):
|
|
|
148
148
|
update: bool = False,
|
|
149
149
|
clear: bool = False,
|
|
150
150
|
extra_data: dict | None = None,
|
|
151
|
-
) ->
|
|
151
|
+
) -> httpx.Response:
|
|
152
152
|
"""
|
|
153
153
|
Execute archivebox schedule command
|
|
154
154
|
|
|
@@ -206,7 +206,7 @@ class Api(BaseApiClient):
|
|
|
206
206
|
as_csv: str | bool = "timestamp,url",
|
|
207
207
|
with_headers: bool = False,
|
|
208
208
|
extra_data: dict | None = None,
|
|
209
|
-
) ->
|
|
209
|
+
) -> httpx.Response:
|
|
210
210
|
"""
|
|
211
211
|
Execute archivebox list command
|
|
212
212
|
|
|
@@ -265,7 +265,7 @@ class Api(BaseApiClient):
|
|
|
265
265
|
filter_type: str = "exact",
|
|
266
266
|
filter_patterns: list[str] | None = None,
|
|
267
267
|
extra_data: dict | None = None,
|
|
268
|
-
) ->
|
|
268
|
+
) -> httpx.Response:
|
|
269
269
|
"""
|
|
270
270
|
Execute archivebox remove command
|
|
271
271
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
from typing import Any
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import httpx
|
|
4
4
|
from agent_connector_sdk.exceptions import (
|
|
5
5
|
ParameterError,
|
|
6
6
|
require_auth,
|
|
@@ -36,7 +36,7 @@ class Api(BaseApiClient):
|
|
|
36
36
|
offset: int = 0,
|
|
37
37
|
page: int = 0,
|
|
38
38
|
api_key: str | None = None,
|
|
39
|
-
) ->
|
|
39
|
+
) -> httpx.Response:
|
|
40
40
|
"""
|
|
41
41
|
Retrieve list of snapshots
|
|
42
42
|
|
|
@@ -109,7 +109,7 @@ class Api(BaseApiClient):
|
|
|
109
109
|
@require_auth
|
|
110
110
|
def get_snapshot(
|
|
111
111
|
self, snapshot_id: str, with_archiveresults: bool = True
|
|
112
|
-
) ->
|
|
112
|
+
) -> httpx.Response:
|
|
113
113
|
"""
|
|
114
114
|
Get a specific Snapshot by abid or id
|
|
115
115
|
|
|
@@ -154,7 +154,7 @@ class Api(BaseApiClient):
|
|
|
154
154
|
offset: int = 0,
|
|
155
155
|
page: int = 0,
|
|
156
156
|
api_key: str | None = None,
|
|
157
|
-
) ->
|
|
157
|
+
) -> httpx.Response:
|
|
158
158
|
"""
|
|
159
159
|
List all ArchiveResult entries matching these filters
|
|
160
160
|
|
|
@@ -217,7 +217,7 @@ class Api(BaseApiClient):
|
|
|
217
217
|
return response
|
|
218
218
|
|
|
219
219
|
@require_auth
|
|
220
|
-
def get_archiveresult(self, archiveresult_id: str) ->
|
|
220
|
+
def get_archiveresult(self, archiveresult_id: str) -> httpx.Response:
|
|
221
221
|
"""
|
|
222
222
|
Get a specific ArchiveResult by id or abid
|
|
223
223
|
|
|
@@ -246,7 +246,7 @@ class Api(BaseApiClient):
|
|
|
246
246
|
offset: int = 0,
|
|
247
247
|
page: int = 0,
|
|
248
248
|
api_key: str | None = None,
|
|
249
|
-
) ->
|
|
249
|
+
) -> httpx.Response:
|
|
250
250
|
"""
|
|
251
251
|
Retrieve list of tags
|
|
252
252
|
|
|
@@ -277,7 +277,7 @@ class Api(BaseApiClient):
|
|
|
277
277
|
return response
|
|
278
278
|
|
|
279
279
|
@require_auth
|
|
280
|
-
def get_tag(self, tag_id: str, with_snapshots: bool = True) ->
|
|
280
|
+
def get_tag(self, tag_id: str, with_snapshots: bool = True) -> httpx.Response:
|
|
281
281
|
"""
|
|
282
282
|
Get a specific Tag by id or abid
|
|
283
283
|
|
|
@@ -302,7 +302,7 @@ class Api(BaseApiClient):
|
|
|
302
302
|
return response
|
|
303
303
|
|
|
304
304
|
@require_auth
|
|
305
|
-
def get_any(self, abid: str) ->
|
|
305
|
+
def get_any(self, abid: str) -> httpx.Response:
|
|
306
306
|
"""
|
|
307
307
|
Get a specific Snapshot, ArchiveResult, or Tag by abid
|
|
308
308
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"algorithm": "agent-connector-sdk:mcp-tool-contract-compat:v2",
|
|
3
|
-
"connector": "archivebox-
|
|
3
|
+
"connector": "archivebox-mcp",
|
|
4
4
|
"schema_version": "1",
|
|
5
5
|
"tools": {
|
|
6
6
|
"archivebox_get_snapshots": "99b97a41c2cc29078b2cf27af52c6ab99ebc97182eb6100611a86fe3a3abfea4"
|
|
@@ -40,12 +40,12 @@ logger = logging.getLogger("archivebox_api.kg")
|
|
|
40
40
|
_SOURCE = "archivebox-api"
|
|
41
41
|
_DOMAIN = "archivebox"
|
|
42
42
|
|
|
43
|
-
_BINDING = IngestBinding(connector="archivebox-
|
|
43
|
+
_BINDING = IngestBinding(connector="archivebox-mcp", stream=_DOMAIN)
|
|
44
44
|
# sanitize=False: the archived page's own source URL is the product data for a web
|
|
45
45
|
# archive, not incidental PII — the default PersistencePrivacyGuard redacts URL-shaped
|
|
46
46
|
# property values ("[REDACTED_LOCATION]"), which would destroy blob provenance here.
|
|
47
47
|
_BLOB_BINDING = IngestBinding(
|
|
48
|
-
connector="archivebox-
|
|
48
|
+
connector="archivebox-mcp",
|
|
49
49
|
stream=_DOMAIN,
|
|
50
50
|
media_type="AssetOccurrence",
|
|
51
51
|
sanitize=False,
|
|
@@ -66,9 +66,7 @@ def _to_entity(record: dict[str, Any]) -> Entity:
|
|
|
66
66
|
|
|
67
67
|
def _to_relationship(record: dict[str, Any]) -> Relationship:
|
|
68
68
|
props = {
|
|
69
|
-
k: v
|
|
70
|
-
for k, v in record.items()
|
|
71
|
-
if k not in ("source", "target", "relationship")
|
|
69
|
+
k: v for k, v in record.items() if k not in ("source", "target", "relationship")
|
|
72
70
|
}
|
|
73
71
|
return Relationship(
|
|
74
72
|
source=record["source"],
|
|
@@ -331,7 +329,9 @@ async def ingest_snapshots(
|
|
|
331
329
|
if entities
|
|
332
330
|
else None
|
|
333
331
|
)
|
|
334
|
-
doc_res =
|
|
332
|
+
doc_res = (
|
|
333
|
+
await ingest_documents(documents, ingest=ingest) if documents else None
|
|
334
|
+
)
|
|
335
335
|
except IngestUnavailableError as e:
|
|
336
336
|
logger.debug("Operation failed: error_type=%s", type(e).__name__)
|
|
337
337
|
return None
|
|
@@ -5,27 +5,7 @@ Tool Interface & MCP Factory
|
|
|
5
5
|
Provides dynamic tool registration and stdio/SSE/http interfaces using FastMCP.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
import warnings
|
|
9
|
-
|
|
10
|
-
from fastmcp import Context, FastMCP
|
|
11
|
-
from fastmcp.dependencies import Depends
|
|
12
|
-
from fastmcp.utilities.logging import get_logger
|
|
13
|
-
from pydantic import Field
|
|
14
|
-
|
|
15
|
-
# Filter RequestsDependencyWarning early to prevent log spam
|
|
16
8
|
# Telemetry & Observability
|
|
17
|
-
with warnings.catch_warnings():
|
|
18
|
-
warnings.simplefilter("ignore")
|
|
19
|
-
try:
|
|
20
|
-
from requests.exceptions import RequestsDependencyWarning
|
|
21
|
-
|
|
22
|
-
warnings.filterwarnings("ignore", category=RequestsDependencyWarning)
|
|
23
|
-
except ImportError:
|
|
24
|
-
pass
|
|
25
|
-
|
|
26
|
-
warnings.filterwarnings("ignore", message=".*urllib3.*or chardet.*")
|
|
27
|
-
warnings.filterwarnings("ignore", message=".*urllib3.*or charset_normalizer.*")
|
|
28
|
-
|
|
29
9
|
import logging
|
|
30
10
|
import sys
|
|
31
11
|
from typing import Any
|
|
@@ -35,13 +15,17 @@ from agent_connector_sdk.mcp.action_dispatch import resolve_action
|
|
|
35
15
|
from agent_connector_sdk.mcp.concurrency import run_blocking
|
|
36
16
|
from agent_connector_sdk.mcp.server import create_mcp_server
|
|
37
17
|
from agent_connector_sdk.mcp.tool_surface import register_tool_surface
|
|
18
|
+
from fastmcp import Context, FastMCP
|
|
19
|
+
from fastmcp.dependencies import Depends
|
|
20
|
+
from fastmcp.utilities.logging import get_logger
|
|
21
|
+
from pydantic import Field
|
|
38
22
|
from starlette.requests import Request
|
|
39
23
|
from starlette.responses import JSONResponse
|
|
40
24
|
|
|
41
25
|
from archivebox_api.api_client import Api
|
|
42
26
|
from archivebox_api.auth import get_client
|
|
43
27
|
|
|
44
|
-
__version__ = "2.5.
|
|
28
|
+
__version__ = "2.5.3"
|
|
45
29
|
|
|
46
30
|
# Telemetry & Observability
|
|
47
31
|
logger = get_logger(name="archivebox-api")
|
|
@@ -386,7 +370,7 @@ def get_mcp_instance() -> tuple[Any, ...]:
|
|
|
386
370
|
version=__version__,
|
|
387
371
|
instructions="archivebox-api MCP Server — one typed tool per API operation.",
|
|
388
372
|
content=ConnectorContent(
|
|
389
|
-
connector="archivebox-
|
|
373
|
+
connector="archivebox-mcp",
|
|
390
374
|
package_root=package_root,
|
|
391
375
|
package_version=__version__,
|
|
392
376
|
manifest_path=package_root / "connector_manifest.yml",
|
|
@@ -24,9 +24,12 @@ import threading
|
|
|
24
24
|
import time
|
|
25
25
|
from collections.abc import Callable
|
|
26
26
|
from dataclasses import dataclass, field
|
|
27
|
-
from typing import Any
|
|
27
|
+
from typing import Any, cast
|
|
28
28
|
|
|
29
|
-
import
|
|
29
|
+
import httpx
|
|
30
|
+
from agent_connector_sdk.http.client import create_http_client
|
|
31
|
+
from agent_connector_sdk.http.options import HttpClientOptions
|
|
32
|
+
from agent_connector_sdk.tls.resolve import resolve_tls_profile
|
|
30
33
|
|
|
31
34
|
__all__ = [
|
|
32
35
|
"REFUSAL_REASONS",
|
|
@@ -76,12 +79,22 @@ class ArchiveSession:
|
|
|
76
79
|
base_url: str
|
|
77
80
|
username: str
|
|
78
81
|
password: str
|
|
79
|
-
verify: bool | str = True
|
|
80
82
|
timeout_s: float = 30.0
|
|
81
|
-
session:
|
|
83
|
+
session: httpx.Client = field(default=cast("httpx.Client", None))
|
|
82
84
|
_logged_in: bool = field(default=False, init=False)
|
|
83
85
|
_lock: threading.Lock = field(default_factory=threading.Lock, init=False)
|
|
84
86
|
|
|
87
|
+
def __post_init__(self) -> None:
|
|
88
|
+
if self.session is None:
|
|
89
|
+
self.session = create_http_client(
|
|
90
|
+
HttpClientOptions(
|
|
91
|
+
base_url=self.base_url.rstrip("/"),
|
|
92
|
+
timeout=self.timeout_s,
|
|
93
|
+
tls=resolve_tls_profile("archivebox"),
|
|
94
|
+
allow_plaintext=self.base_url.startswith("http://"),
|
|
95
|
+
)
|
|
96
|
+
)
|
|
97
|
+
|
|
85
98
|
def _url(self, path: str) -> str:
|
|
86
99
|
return self.base_url.rstrip("/") + path
|
|
87
100
|
|
|
@@ -90,9 +103,7 @@ class ArchiveSession:
|
|
|
90
103
|
raise WebSourceRefused(
|
|
91
104
|
"upstream_failed", self.base_url, "archive login is not configured"
|
|
92
105
|
)
|
|
93
|
-
page = self.session.get(
|
|
94
|
-
self._url(_LOGIN_PATH), timeout=self.timeout_s, verify=self.verify
|
|
95
|
-
)
|
|
106
|
+
page = self.session.get(self._url(_LOGIN_PATH))
|
|
96
107
|
match = _CSRF_RE.search(page.text)
|
|
97
108
|
if match is None:
|
|
98
109
|
raise WebSourceRefused("upstream_failed", self.base_url, "no login form")
|
|
@@ -105,16 +116,13 @@ class ArchiveSession:
|
|
|
105
116
|
"next": "/",
|
|
106
117
|
},
|
|
107
118
|
headers={"Referer": self._url(_LOGIN_PATH)},
|
|
108
|
-
allow_redirects=False,
|
|
109
|
-
timeout=self.timeout_s,
|
|
110
|
-
verify=self.verify,
|
|
111
119
|
)
|
|
112
120
|
location = response.headers.get("location", "")
|
|
113
121
|
if response.status_code != 302 or _LOGIN_PATH in location:
|
|
114
122
|
raise WebSourceRefused("upstream_failed", self.base_url, "login refused")
|
|
115
123
|
self._logged_in = True
|
|
116
124
|
|
|
117
|
-
def _is_login_redirect(self, response:
|
|
125
|
+
def _is_login_redirect(self, response: httpx.Response) -> bool:
|
|
118
126
|
return response.is_redirect and _LOGIN_PATH in response.headers.get(
|
|
119
127
|
"location", ""
|
|
120
128
|
)
|
|
@@ -136,27 +144,23 @@ class ArchiveSession:
|
|
|
136
144
|
with self._lock:
|
|
137
145
|
if not self._logged_in:
|
|
138
146
|
self.login()
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
147
|
+
response = self.session.get(self._url(path))
|
|
148
|
+
if self._is_login_redirect(response):
|
|
149
|
+
# The archive session expired: log in again and read once more.
|
|
150
|
+
with self._lock:
|
|
151
|
+
self.login()
|
|
152
|
+
response = self.session.get(self._url(path))
|
|
153
|
+
if self._is_login_redirect(response):
|
|
154
|
+
raise WebSourceRefused("upstream_failed", path, "session expired")
|
|
155
|
+
if response.status_code == 404:
|
|
156
|
+
return None
|
|
157
|
+
if response.status_code != 200:
|
|
158
|
+
raise WebSourceRefused(
|
|
159
|
+
"upstream_failed", path, f"HTTP {response.status_code}"
|
|
145
160
|
)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
continue
|
|
150
|
-
if response.status_code == 404:
|
|
151
|
-
return None
|
|
152
|
-
if response.status_code != 200:
|
|
153
|
-
raise WebSourceRefused(
|
|
154
|
-
"upstream_failed", path, f"HTTP {response.status_code}"
|
|
155
|
-
)
|
|
156
|
-
if len(response.content) > max_bytes:
|
|
157
|
-
raise WebSourceRefused("size_exceeded", path)
|
|
158
|
-
return response.headers.get("content-type", ""), response.content
|
|
159
|
-
raise WebSourceRefused("upstream_failed", path, "session expired")
|
|
161
|
+
if len(response.content) > max_bytes:
|
|
162
|
+
raise WebSourceRefused("size_exceeded", path)
|
|
163
|
+
return response.headers.get("content-type", ""), response.content
|
|
160
164
|
|
|
161
165
|
|
|
162
166
|
def _json(response: Any) -> Any:
|
|
@@ -16,8 +16,11 @@ import asyncio
|
|
|
16
16
|
import json
|
|
17
17
|
import re
|
|
18
18
|
from typing import Any
|
|
19
|
+
from urllib.parse import urlsplit
|
|
19
20
|
|
|
20
|
-
import
|
|
21
|
+
import httpx
|
|
22
|
+
from agent_connector_sdk.http.client import create_http_client
|
|
23
|
+
from agent_connector_sdk.http.options import HttpClientOptions
|
|
21
24
|
from agent_connector_sdk.ports.web_source import (
|
|
22
25
|
DEFAULT_MIN_INTERVAL_S,
|
|
23
26
|
DomainRateLimiter,
|
|
@@ -47,9 +50,18 @@ _TAG = re.compile(r"<[^>]+>")
|
|
|
47
50
|
|
|
48
51
|
|
|
49
52
|
def _read_robots(url: str) -> tuple[int, str]:
|
|
53
|
+
parts = urlsplit(url)
|
|
50
54
|
try:
|
|
51
|
-
|
|
52
|
-
|
|
55
|
+
with create_http_client(
|
|
56
|
+
HttpClientOptions(
|
|
57
|
+
base_url=f"{parts.scheme}://{parts.netloc}",
|
|
58
|
+
timeout=15,
|
|
59
|
+
headers={"User-Agent": USER_AGENT},
|
|
60
|
+
allow_plaintext=True,
|
|
61
|
+
)
|
|
62
|
+
) as client:
|
|
63
|
+
response = client.get(url)
|
|
64
|
+
except (httpx.HTTPError, ValueError) as exc:
|
|
53
65
|
raise WebSourceRefused("upstream_failed", type(exc).__name__) from exc
|
|
54
66
|
return response.status_code, response.text
|
|
55
67
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: archivebox-api
|
|
3
|
-
Version: 2.5.
|
|
3
|
+
Version: 2.5.3
|
|
4
4
|
Summary: Pythonic ArchiveBox API Wrapper and Fast MCP Server for Agentic AI use!
|
|
5
5
|
Author-email: Project Maintainers <maintainers@users.noreply.github.com>
|
|
6
6
|
License: MIT
|
|
@@ -12,12 +12,11 @@ Classifier: Programming Language :: Python :: 3
|
|
|
12
12
|
Requires-Python: <3.15,>=3.12
|
|
13
13
|
Description-Content-Type: text/markdown
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.
|
|
16
|
-
Requires-Dist: requests>=2.34.2
|
|
15
|
+
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.11
|
|
17
16
|
Provides-Extra: mcp
|
|
18
|
-
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.
|
|
17
|
+
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.11; extra == "mcp"
|
|
19
18
|
Provides-Extra: all
|
|
20
|
-
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.
|
|
19
|
+
Requires-Dist: agent-connector-sdk[mcp]<1.0.0,>=0.3.11; extra == "all"
|
|
21
20
|
Provides-Extra: test
|
|
22
21
|
Requires-Dist: pytest-xdist>=3.8.0; extra == "test"
|
|
23
22
|
Requires-Dist: pytest>=9.1.1; extra == "test"
|
|
@@ -47,7 +46,7 @@ Dynamic: license-file
|
|
|
47
46
|

|
|
48
47
|

|
|
49
48
|
|
|
50
|
-
*Version: 2.5.
|
|
49
|
+
*Version: 2.5.3*
|
|
51
50
|
|
|
52
51
|
> **Documentation** — Installation, deployment, usage across the API, CLI, MCP, and
|
|
53
52
|
> A2A agent interfaces, and guidance for provisioning the ArchiveBox platform are
|
|
@@ -4,12 +4,12 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "archivebox-api"
|
|
7
|
-
version = "2.5.
|
|
7
|
+
version = "2.5.3"
|
|
8
8
|
description = "Pythonic ArchiveBox API Wrapper and Fast MCP Server for Agentic AI use!"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
classifiers = [ "Development Status :: 5 - Production/Stable", "License :: Public Domain", "Environment :: Console", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3",]
|
|
11
11
|
requires-python = ">=3.12, <3.15"
|
|
12
|
-
dependencies = [ "agent-connector-sdk[mcp]>=0.3.
|
|
12
|
+
dependencies = [ "agent-connector-sdk[mcp]>=0.3.11,<1.0.0",]
|
|
13
13
|
[[project.authors]]
|
|
14
14
|
name = "Project Maintainers"
|
|
15
15
|
email = "maintainers@users.noreply.github.com"
|
|
@@ -18,14 +18,18 @@ email = "maintainers@users.noreply.github.com"
|
|
|
18
18
|
text = "MIT"
|
|
19
19
|
|
|
20
20
|
[project.optional-dependencies]
|
|
21
|
-
mcp = [ "agent-connector-sdk[mcp]>=0.3.
|
|
22
|
-
all = [ "agent-connector-sdk[mcp]>=0.3.
|
|
21
|
+
mcp = [ "agent-connector-sdk[mcp]>=0.3.11,<1.0.0",]
|
|
22
|
+
all = [ "agent-connector-sdk[mcp]>=0.3.11,<1.0.0",]
|
|
23
23
|
test = [
|
|
24
24
|
"pytest-xdist>=3.8.0", "pytest>=9.1.1", "pytest-asyncio>=1.4.0", "pytest-cov>=7.1.0",]
|
|
25
25
|
|
|
26
26
|
[project.scripts]
|
|
27
27
|
archivebox-mcp = "archivebox_api.mcp_server:mcp_server"
|
|
28
28
|
|
|
29
|
+
# The one declared connector identity: the fleet name (SDK-CONNECTOR-CONTROL-R071).
|
|
30
|
+
[project.entry-points."agent_connector_sdk.connector"]
|
|
31
|
+
archivebox-mcp = "archivebox_api"
|
|
32
|
+
|
|
29
33
|
[project.entry-points."agent_connector_sdk.skill_providers"]
|
|
30
34
|
archivebox-api = "archivebox_api.skills"
|
|
31
35
|
|