archivebox-api 2.5.9__tar.gz → 2.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/CHANGELOG.md +18 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/PKG-INFO +1 -1
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/__init__.py +2 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/api/api_client_auth.py +10 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/api/api_client_base.py +61 -0
- archivebox_api-2.6.0/archivebox_api/api/api_client_cli.py +350 -0
- archivebox_api-2.6.0/archivebox_api/api/api_client_core.py +553 -0
- archivebox_api-2.6.0/archivebox_api/api/api_client_crawls.py +357 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/api_client.py +2 -1
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/connector_sources.py +21 -15
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/kg_ingest.py +21 -18
- archivebox_api-2.6.0/archivebox_api/mcp_server.py +213 -0
- archivebox_api-2.6.0/archivebox_api/ontology/access_contracts.ttl +70 -0
- archivebox_api-2.6.0/archivebox_api/ontology/archivebox.ttl +147 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/ontology/certification.json +5 -5
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/ontology/fixtures/records.json +1 -1
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/ontology/mappings/source.yaml +2 -4
- archivebox_api-2.6.0/archivebox_api/ontology/pack.ttl +376 -0
- archivebox_api-2.6.0/archivebox_api/ontology/shapes/connector.shacl.ttl +84 -0
- archivebox_api-2.6.0/archivebox_api/prompts/archivebox_operator.json +93 -0
- archivebox_api-2.6.0/archivebox_api/prompts/archivebox_research_archivist.json +57 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/skills/archivebox-api-development/SKILL.md +11 -7
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/skills/archivebox-api-operations/SKILL.md +2 -2
- archivebox_api-2.6.0/archivebox_api/skills/archivebox-api-operations/references/catalog.md +8 -0
- archivebox_api-2.6.0/archivebox_api/skills/archivebox-archive-administration/SKILL.md +58 -0
- archivebox_api-2.6.0/archivebox_api/skills/archivebox-kg-ingestion/SKILL.md +41 -0
- archivebox_api-2.6.0/archivebox_api/skills/archivebox-research-archive/SKILL.md +88 -0
- archivebox_api-2.6.0/archivebox_api/skills/archivebox-snapshot-search/SKILL.md +54 -0
- archivebox_api-2.6.0/archivebox_api/skills/archivebox-web-archiving/SKILL.md +69 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/web_source.py +1 -1
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api.egg-info/PKG-INFO +1 -1
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api.egg-info/SOURCES.txt +8 -5
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/pyproject.toml +1 -4
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/release-readiness.json +94 -92
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/conftest.py +1 -1
- archivebox_api-2.6.0/tests/test_archivebox_coverage_extra.py +169 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_conformance.py +1 -1
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_kg_ingest.py +12 -14
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_kg_media.py +2 -2
- archivebox_api-2.6.0/tests/test_surface_contracts.py +239 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_web_source.py +2 -2
- archivebox_api-2.5.9/archivebox_api/api/api_client_cli.py +0 -306
- archivebox_api-2.5.9/archivebox_api/api/api_client_core.py +0 -325
- archivebox_api-2.5.9/archivebox_api/mcp_server.py +0 -348
- archivebox_api-2.5.9/archivebox_api/ontology/archivebox.ttl +0 -101
- archivebox_api-2.5.9/archivebox_api/ontology/shapes/connector.shacl.ttl +0 -44
- archivebox_api-2.5.9/archivebox_api/prompts/web_archivist_specialist.json +0 -41
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-api-operations/references/catalog.md +0 -7
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-kg-ingestion/SKILL.md +0 -91
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-kg-ingestion/WORKFLOW.md +0 -77
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-snapshot-search/SKILL.md +0 -94
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-snapshot-search/WORKFLOW.md +0 -80
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-web-archiving/SKILL.md +0 -96
- archivebox_api-2.5.9/archivebox_api/skills/archivebox-web-archiving/WORKFLOW.md +0 -82
- archivebox_api-2.5.9/tests/test_action_discovery.py +0 -67
- archivebox_api-2.5.9/tests/test_archivebox_coverage_extra.py +0 -384
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/.pre-commit-config.yaml +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/AGENTS.md +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/CLAUDE.md +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/LICENSE +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/MANIFEST.in +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/README.md +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/__main__.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/api/__init__.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/auth.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/mcp_config.json +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/ontology/__init__.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/ontology/migrations/manifest.json +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/skills/archivebox-api-operations/agents/openai.yaml +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api/web_source_contract.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api.egg-info/dependency_links.txt +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api.egg-info/entry_points.txt +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api.egg-info/requires.txt +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/archivebox_api.egg-info/top_level.txt +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/mkdocs.yml +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/requirements.txt +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/setup.cfg +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_archivebox_api_brute_force_coverage.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_archivebox_coverage.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_archivebox_development_skill.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_basic.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_init_dynamics.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_publication_workflow.py +0 -0
- {archivebox_api-2.5.9 → archivebox_api-2.6.0}/tests/test_sdk_transport.py +0 -0
|
@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [2.6.0] - 2026-11-06
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- One typed tool per ArchiveBox 0.9 REST operation (46 operations, 48 operation tools): adds snapshot create/patch/delete and RSS, archive-result create/patch with file upload, tag create/rename/delete/search/autocomplete/export and snapshot tagging, crawls and crawl files, machines, binaries and personas, and the browser session exchange. Parameters follow the 0.9 API (`plugins`, `persona`, `max_urls`, `output_str`, `id`).
|
|
12
|
+
- `archivebox_cli_list` is replaced by `archivebox_cli_search` (`/api/v1/cli/list` no longer exists).
|
|
13
|
+
- Tool results return the upstream JSON or text body; failed calls return `{status_code, error}`. Before, every tool returned only `{status_code}`.
|
|
14
|
+
- `archivebox_cli_remove` refuses an empty selection.
|
|
15
|
+
- KG ingest and the web source read 0.9 record shapes; KG node types are the core `WebSnapshot` and `ArchiveResult`.
|
|
16
|
+
- The ingest tools take typed filters instead of `params_json`.
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
- Ontology: `pack.ttl` (tool-use contracts, feature/skill/prompt links, ingest contract), `access_contracts.ttl` (data location, routing, federation); the domain classes roll up to medulla core by `rdfs:subClassOf` and no longer redeclare core terms.
|
|
20
|
+
- Skills `archivebox-research-archive` and `archivebox-archive-administration`; the other skills are rewritten for the typed tools.
|
|
21
|
+
- Agent-definition prompts `archivebox_operator` and `archivebox_research_archivist`.
|
|
22
|
+
|
|
23
|
+
### Removed
|
|
24
|
+
- The action-routed `archivebox_core`, `archivebox_cli` and `archivebox_authentication` registrars and the `web_archivist_specialist` prompt.
|
|
25
|
+
|
|
8
26
|
## [2.5.9] - 2026-10-30
|
|
9
27
|
|
|
10
28
|
### Changed
|
|
@@ -91,10 +91,12 @@ from scion_sdk.connector import Connector
|
|
|
91
91
|
|
|
92
92
|
from archivebox_api.api_client import Api
|
|
93
93
|
from archivebox_api.auth import get_client
|
|
94
|
+
from archivebox_api.mcp_server import coerce_result
|
|
94
95
|
|
|
95
96
|
connector = Connector(
|
|
96
97
|
api=Api,
|
|
97
98
|
get_client=get_client,
|
|
98
99
|
tools="archivebox_api.mcp_server",
|
|
100
|
+
result_coercer=coerce_result,
|
|
99
101
|
instructions="archivebox-api MCP Server — one typed tool per API operation.",
|
|
100
102
|
)
|
|
@@ -4,6 +4,7 @@ import httpx
|
|
|
4
4
|
from pydantic import ValidationError
|
|
5
5
|
from scion_sdk.exceptions import (
|
|
6
6
|
ParameterError,
|
|
7
|
+
require_auth,
|
|
7
8
|
)
|
|
8
9
|
|
|
9
10
|
from archivebox_api.api.api_client_base import BaseApiClient
|
|
@@ -63,3 +64,12 @@ class Api(BaseApiClient):
|
|
|
63
64
|
except ValidationError as e:
|
|
64
65
|
raise ParameterError(f"Invalid parameters: {e.errors()}") from e
|
|
65
66
|
return response
|
|
67
|
+
|
|
68
|
+
@require_auth
|
|
69
|
+
def browser_session(self) -> httpx.Response:
|
|
70
|
+
"""POST /api/v1/auth/browser_session
|
|
71
|
+
|
|
72
|
+
Exchange an administrator API key for a normal browser session (the
|
|
73
|
+
response sets the session cookie used by the web UI and file views).
|
|
74
|
+
"""
|
|
75
|
+
return self._send("POST", "/api/v1/auth/browser_session", json={})
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import sys
|
|
2
2
|
from abc import ABC, abstractmethod
|
|
3
|
+
from typing import Any
|
|
3
4
|
|
|
4
5
|
import httpx
|
|
5
6
|
from scion_sdk.exceptions import (
|
|
@@ -78,6 +79,66 @@ class BaseApiClient(ABC):
|
|
|
78
79
|
raise AuthError("Failed to retrieve API token")
|
|
79
80
|
self.headers["Authorization"] = f"Bearer {fetched_token}"
|
|
80
81
|
|
|
82
|
+
def _send(
|
|
83
|
+
self,
|
|
84
|
+
verb: str,
|
|
85
|
+
path: str,
|
|
86
|
+
*,
|
|
87
|
+
params: dict[str, Any] | None = None,
|
|
88
|
+
json: dict[str, Any] | None = None,
|
|
89
|
+
api_key: str | None = None,
|
|
90
|
+
follow_redirects: bool = False,
|
|
91
|
+
) -> httpx.Response:
|
|
92
|
+
"""Send one ArchiveBox REST call; ``None`` values are dropped from both
|
|
93
|
+
the query string and the JSON body so upstream defaults apply."""
|
|
94
|
+
query = {k: v for k, v in (params or {}).items() if v is not None}
|
|
95
|
+
if api_key:
|
|
96
|
+
query["api_key"] = api_key
|
|
97
|
+
body = None
|
|
98
|
+
if json is not None:
|
|
99
|
+
body = {k: v for k, v in json.items() if v is not None}
|
|
100
|
+
return self._session.request(
|
|
101
|
+
verb,
|
|
102
|
+
f"{self.url}{path}",
|
|
103
|
+
params=query or None,
|
|
104
|
+
json=body,
|
|
105
|
+
headers=self.headers,
|
|
106
|
+
follow_redirects=follow_redirects,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
def _send_form(
|
|
110
|
+
self,
|
|
111
|
+
verb: str,
|
|
112
|
+
path: str,
|
|
113
|
+
*,
|
|
114
|
+
fields: dict[str, Any],
|
|
115
|
+
files: dict[str, str] | None = None,
|
|
116
|
+
) -> httpx.Response:
|
|
117
|
+
"""Send a multipart form (ArchiveBox's archive-result upload shape).
|
|
118
|
+
|
|
119
|
+
``files`` maps each output path (relative to the plugin directory) to
|
|
120
|
+
its base64-encoded content; each becomes one ``files`` part with its
|
|
121
|
+
path in ``output_paths``.
|
|
122
|
+
"""
|
|
123
|
+
import base64
|
|
124
|
+
import json as _json
|
|
125
|
+
|
|
126
|
+
data = {k: str(v) for k, v in fields.items() if v is not None}
|
|
127
|
+
parts: list[tuple[str, tuple[str, bytes]]] = []
|
|
128
|
+
if files:
|
|
129
|
+
data["output_paths"] = _json.dumps(list(files))
|
|
130
|
+
for out_path, b64 in files.items():
|
|
131
|
+
name = out_path.rsplit("/", 1)[-1]
|
|
132
|
+
parts.append(("files", (name, base64.b64decode(b64))))
|
|
133
|
+
headers = {k: v for k, v in self.headers.items() if k != "Content-Type"}
|
|
134
|
+
return self._session.request(
|
|
135
|
+
verb,
|
|
136
|
+
f"{self.url}{path}",
|
|
137
|
+
data=data,
|
|
138
|
+
files=parts or None,
|
|
139
|
+
headers=headers,
|
|
140
|
+
)
|
|
141
|
+
|
|
81
142
|
def close(self) -> None:
|
|
82
143
|
"""Release transport resources and runtime-only TLS material."""
|
|
83
144
|
self._session.close()
|
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
"""ArchiveBox ``/api/v1/cli`` operations: add, update, schedule, search, remove.
|
|
2
|
+
|
|
3
|
+
One client method per upstream operation (CONN-ARCHIVEBOX-API-CONNECTOR-CONTENT-R003).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import httpx
|
|
7
|
+
from scion_sdk.exceptions import ParameterError, require_auth
|
|
8
|
+
|
|
9
|
+
from archivebox_api.api.api_client_base import BaseApiClient
|
|
10
|
+
|
|
11
|
+
_CLI = "/api/v1/cli"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _selection(
|
|
15
|
+
*,
|
|
16
|
+
after: float | None,
|
|
17
|
+
before: float | None,
|
|
18
|
+
filter_type: str | None,
|
|
19
|
+
filter_patterns: list[str] | None,
|
|
20
|
+
status: str | None,
|
|
21
|
+
url__icontains: str | None,
|
|
22
|
+
url__istartswith: str | None,
|
|
23
|
+
tag: str | None,
|
|
24
|
+
crawl_id: str | None,
|
|
25
|
+
limit: int | None,
|
|
26
|
+
sort: str | None,
|
|
27
|
+
search: str | None,
|
|
28
|
+
) -> dict[str, object]:
|
|
29
|
+
"""The snapshot-selection fields shared by update, search and remove."""
|
|
30
|
+
return {
|
|
31
|
+
"after": after,
|
|
32
|
+
"before": before,
|
|
33
|
+
"filter_type": filter_type,
|
|
34
|
+
"filter_patterns": filter_patterns,
|
|
35
|
+
"status": status,
|
|
36
|
+
"url__icontains": url__icontains,
|
|
37
|
+
"url__istartswith": url__istartswith,
|
|
38
|
+
"tag": tag,
|
|
39
|
+
"crawl_id": crawl_id,
|
|
40
|
+
"limit": limit,
|
|
41
|
+
"sort": sort,
|
|
42
|
+
"search": search,
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Api(BaseApiClient):
|
|
47
|
+
@require_auth
|
|
48
|
+
def cli_add(
|
|
49
|
+
self,
|
|
50
|
+
urls: list[str],
|
|
51
|
+
tag: str = "",
|
|
52
|
+
depth: int = 0,
|
|
53
|
+
snapshot_ids: list[str] | None = None,
|
|
54
|
+
max_urls: int = 0,
|
|
55
|
+
crawl_max_size: int = 0,
|
|
56
|
+
crawl_timeout: int = 0,
|
|
57
|
+
snapshot_max_size: int = 0,
|
|
58
|
+
parser: str = "auto",
|
|
59
|
+
plugins: str = "",
|
|
60
|
+
persona: str = "Default",
|
|
61
|
+
only_new: bool | None = None,
|
|
62
|
+
update: bool = False,
|
|
63
|
+
overwrite: bool = False,
|
|
64
|
+
index_only: bool = False,
|
|
65
|
+
) -> httpx.Response:
|
|
66
|
+
"""POST /api/v1/cli/add
|
|
67
|
+
|
|
68
|
+
Add URLs to the archive (one crawl; one snapshot per URL) and queue them
|
|
69
|
+
for capture.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
urls: URLs (or text/feeds the parser extracts URLs from) to archive.
|
|
73
|
+
tag: Comma-separated tag names applied to every new snapshot.
|
|
74
|
+
depth: Crawl depth (0 = only these URLs, 1 = also their outlinks).
|
|
75
|
+
snapshot_ids: Re-run capture on these existing snapshots instead.
|
|
76
|
+
max_urls: Cap on URLs the crawl may queue (0 = unlimited).
|
|
77
|
+
crawl_max_size: Cap on total crawl bytes (0 = unlimited).
|
|
78
|
+
crawl_timeout: Crawl time limit in seconds (0 = unlimited).
|
|
79
|
+
snapshot_max_size: Cap on bytes per snapshot (0 = unlimited).
|
|
80
|
+
parser: Input parser (``auto``, ``txt``, ``rss``, ``html``, ...).
|
|
81
|
+
plugins: Comma-separated plugins to run (empty = instance defaults),
|
|
82
|
+
for example ``wget,singlefile,pdf,screenshot,readability``.
|
|
83
|
+
persona: Browser persona (cookies/profile) to capture with.
|
|
84
|
+
only_new: Skip URLs already in the archive.
|
|
85
|
+
update: Re-archive URLs already in the archive.
|
|
86
|
+
overwrite: Overwrite existing outputs on re-archive.
|
|
87
|
+
index_only: Add to the index without capturing.
|
|
88
|
+
"""
|
|
89
|
+
body = {
|
|
90
|
+
"urls": urls,
|
|
91
|
+
"snapshot_ids": snapshot_ids,
|
|
92
|
+
"tag": tag,
|
|
93
|
+
"depth": depth,
|
|
94
|
+
"max_urls": max_urls,
|
|
95
|
+
"crawl_max_size": crawl_max_size,
|
|
96
|
+
"crawl_timeout": crawl_timeout,
|
|
97
|
+
"snapshot_max_size": snapshot_max_size,
|
|
98
|
+
"parser": parser,
|
|
99
|
+
"plugins": plugins,
|
|
100
|
+
"persona": persona,
|
|
101
|
+
"only_new": only_new,
|
|
102
|
+
"update": update,
|
|
103
|
+
"overwrite": overwrite,
|
|
104
|
+
"index_only": index_only,
|
|
105
|
+
}
|
|
106
|
+
return self._send("POST", f"{_CLI}/add", json=body)
|
|
107
|
+
|
|
108
|
+
@require_auth
|
|
109
|
+
def cli_update(
|
|
110
|
+
self,
|
|
111
|
+
after: float | None = 0,
|
|
112
|
+
before: float | None = None,
|
|
113
|
+
filter_type: str | None = "substring",
|
|
114
|
+
filter_patterns: list[str] | None = None,
|
|
115
|
+
status: str | None = None,
|
|
116
|
+
url__icontains: str | None = None,
|
|
117
|
+
url__istartswith: str | None = None,
|
|
118
|
+
tag: str | None = None,
|
|
119
|
+
crawl_id: str | None = None,
|
|
120
|
+
limit: int | None = None,
|
|
121
|
+
sort: str | None = None,
|
|
122
|
+
search: str | None = None,
|
|
123
|
+
resume: str | None = None,
|
|
124
|
+
batch_size: int = 100,
|
|
125
|
+
continuous: bool = False,
|
|
126
|
+
index_only: bool = False,
|
|
127
|
+
migrate_only: bool = False,
|
|
128
|
+
) -> httpx.Response:
|
|
129
|
+
"""POST /api/v1/cli/update
|
|
130
|
+
|
|
131
|
+
Re-archive (or re-index) the snapshots the selection matches.
|
|
132
|
+
|
|
133
|
+
Args:
|
|
134
|
+
after: Only snapshots bookmarked after this Unix time.
|
|
135
|
+
before: Only snapshots bookmarked before this Unix time.
|
|
136
|
+
filter_type: How ``filter_patterns`` match (exact, substring,
|
|
137
|
+
domain, regex, tag, search).
|
|
138
|
+
filter_patterns: Patterns to match.
|
|
139
|
+
status: Only snapshots with this status.
|
|
140
|
+
url__icontains: URL contains.
|
|
141
|
+
url__istartswith: URL starts with.
|
|
142
|
+
tag: Only snapshots with this tag.
|
|
143
|
+
crawl_id: Only snapshots of this crawl.
|
|
144
|
+
limit: Maximum snapshots.
|
|
145
|
+
sort: Sort field.
|
|
146
|
+
search: Full-text search.
|
|
147
|
+
resume: Resume from this snapshot timestamp.
|
|
148
|
+
batch_size: Snapshots per batch.
|
|
149
|
+
continuous: Keep running and pick up new snapshots.
|
|
150
|
+
index_only: Rebuild the index only.
|
|
151
|
+
migrate_only: Run data migrations only.
|
|
152
|
+
"""
|
|
153
|
+
body = _selection(
|
|
154
|
+
after=after,
|
|
155
|
+
before=before,
|
|
156
|
+
filter_type=filter_type,
|
|
157
|
+
filter_patterns=filter_patterns,
|
|
158
|
+
status=status,
|
|
159
|
+
url__icontains=url__icontains,
|
|
160
|
+
url__istartswith=url__istartswith,
|
|
161
|
+
tag=tag,
|
|
162
|
+
crawl_id=crawl_id,
|
|
163
|
+
limit=limit,
|
|
164
|
+
sort=sort,
|
|
165
|
+
search=search,
|
|
166
|
+
) | {
|
|
167
|
+
"resume": resume,
|
|
168
|
+
"batch_size": batch_size,
|
|
169
|
+
"continuous": continuous,
|
|
170
|
+
"index_only": index_only,
|
|
171
|
+
"migrate_only": migrate_only,
|
|
172
|
+
}
|
|
173
|
+
return self._send("POST", f"{_CLI}/update", json=body)
|
|
174
|
+
|
|
175
|
+
@require_auth
|
|
176
|
+
def cli_schedule(
|
|
177
|
+
self,
|
|
178
|
+
import_path: str | None = None,
|
|
179
|
+
add: bool = False,
|
|
180
|
+
show: bool = False,
|
|
181
|
+
foreground: bool = False,
|
|
182
|
+
run_all: bool = False,
|
|
183
|
+
quiet: bool = False,
|
|
184
|
+
every: str | None = None,
|
|
185
|
+
tag: str = "",
|
|
186
|
+
depth: int = 0,
|
|
187
|
+
only_new: bool | None = None,
|
|
188
|
+
update: bool = False,
|
|
189
|
+
overwrite: bool = False,
|
|
190
|
+
clear: bool = False,
|
|
191
|
+
) -> httpx.Response:
|
|
192
|
+
"""POST /api/v1/cli/schedule
|
|
193
|
+
|
|
194
|
+
Create, list or clear recurring archive schedules.
|
|
195
|
+
|
|
196
|
+
Args:
|
|
197
|
+
import_path: URL or feed to re-import on each run.
|
|
198
|
+
add: Create a schedule.
|
|
199
|
+
show: List existing schedules.
|
|
200
|
+
foreground: Run the scheduler in the foreground.
|
|
201
|
+
run_all: Run every schedule now.
|
|
202
|
+
quiet: Less output.
|
|
203
|
+
every: Interval (``hour``, ``day``, ``week``, ``month`` or cron).
|
|
204
|
+
tag: Comma-separated tags for snapshots the schedule adds.
|
|
205
|
+
depth: Crawl depth for each run.
|
|
206
|
+
only_new: Only add URLs not yet archived.
|
|
207
|
+
update: Re-archive existing URLs on each run.
|
|
208
|
+
overwrite: Overwrite outputs on re-archive.
|
|
209
|
+
clear: Remove all schedules.
|
|
210
|
+
"""
|
|
211
|
+
body = {
|
|
212
|
+
"import_path": import_path,
|
|
213
|
+
"add": add,
|
|
214
|
+
"show": show,
|
|
215
|
+
"foreground": foreground,
|
|
216
|
+
"run_all": run_all,
|
|
217
|
+
"quiet": quiet,
|
|
218
|
+
"every": every,
|
|
219
|
+
"tag": tag,
|
|
220
|
+
"depth": depth,
|
|
221
|
+
"only_new": only_new,
|
|
222
|
+
"update": update,
|
|
223
|
+
"overwrite": overwrite,
|
|
224
|
+
"clear": clear,
|
|
225
|
+
}
|
|
226
|
+
return self._send("POST", f"{_CLI}/schedule", json=body)
|
|
227
|
+
|
|
228
|
+
@require_auth
|
|
229
|
+
def cli_search(
|
|
230
|
+
self,
|
|
231
|
+
after: float | None = 0,
|
|
232
|
+
before: float | None = None,
|
|
233
|
+
filter_type: str | None = "substring",
|
|
234
|
+
filter_patterns: list[str] | None = None,
|
|
235
|
+
status: str | None = None,
|
|
236
|
+
url__icontains: str | None = None,
|
|
237
|
+
url__istartswith: str | None = None,
|
|
238
|
+
tag: str | None = None,
|
|
239
|
+
crawl_id: str | None = None,
|
|
240
|
+
limit: int | None = None,
|
|
241
|
+
sort: str | None = None,
|
|
242
|
+
search: str | None = None,
|
|
243
|
+
as_json: bool = True,
|
|
244
|
+
as_html: bool = False,
|
|
245
|
+
as_csv: str | None = "timestamp,url",
|
|
246
|
+
with_headers: bool = False,
|
|
247
|
+
) -> httpx.Response:
|
|
248
|
+
"""POST /api/v1/cli/search
|
|
249
|
+
|
|
250
|
+
Search the archive index and export the matches (JSON, HTML or CSV).
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
after: Only snapshots bookmarked after this Unix time.
|
|
254
|
+
before: Only snapshots bookmarked before this Unix time.
|
|
255
|
+
filter_type: How ``filter_patterns`` match (exact, substring,
|
|
256
|
+
domain, regex, tag, search).
|
|
257
|
+
filter_patterns: Patterns to match.
|
|
258
|
+
status: Only snapshots with this status.
|
|
259
|
+
url__icontains: URL contains.
|
|
260
|
+
url__istartswith: URL starts with.
|
|
261
|
+
tag: Only snapshots with this tag.
|
|
262
|
+
crawl_id: Only snapshots of this crawl.
|
|
263
|
+
limit: Maximum snapshots.
|
|
264
|
+
sort: Sort field.
|
|
265
|
+
search: Full-text search.
|
|
266
|
+
as_json: Export as JSON.
|
|
267
|
+
as_html: Export as HTML.
|
|
268
|
+
as_csv: Export as CSV with these columns.
|
|
269
|
+
with_headers: Include headers in the export.
|
|
270
|
+
"""
|
|
271
|
+
body = _selection(
|
|
272
|
+
after=after,
|
|
273
|
+
before=before,
|
|
274
|
+
filter_type=filter_type,
|
|
275
|
+
filter_patterns=filter_patterns,
|
|
276
|
+
status=status,
|
|
277
|
+
url__icontains=url__icontains,
|
|
278
|
+
url__istartswith=url__istartswith,
|
|
279
|
+
tag=tag,
|
|
280
|
+
crawl_id=crawl_id,
|
|
281
|
+
limit=limit,
|
|
282
|
+
sort=sort,
|
|
283
|
+
search=search,
|
|
284
|
+
) | {
|
|
285
|
+
"as_json": as_json,
|
|
286
|
+
"as_html": as_html,
|
|
287
|
+
"as_csv": as_csv,
|
|
288
|
+
"with_headers": with_headers,
|
|
289
|
+
}
|
|
290
|
+
return self._send("POST", f"{_CLI}/search", json=body)
|
|
291
|
+
|
|
292
|
+
@require_auth
|
|
293
|
+
def cli_remove(
|
|
294
|
+
self,
|
|
295
|
+
after: float | None = 0,
|
|
296
|
+
before: float | None = None,
|
|
297
|
+
filter_type: str = "exact",
|
|
298
|
+
filter_patterns: list[str] | None = None,
|
|
299
|
+
status: str | None = None,
|
|
300
|
+
url__icontains: str | None = None,
|
|
301
|
+
url__istartswith: str | None = None,
|
|
302
|
+
tag: str | None = None,
|
|
303
|
+
crawl_id: str | None = None,
|
|
304
|
+
limit: int | None = None,
|
|
305
|
+
sort: str | None = None,
|
|
306
|
+
search: str | None = None,
|
|
307
|
+
timeout: float = 60.0,
|
|
308
|
+
) -> httpx.Response:
|
|
309
|
+
"""POST /api/v1/cli/remove
|
|
310
|
+
|
|
311
|
+
Delete the snapshots the selection matches, and their files. Destructive:
|
|
312
|
+
always pass the narrowest selection (an exact URL, or one tag).
|
|
313
|
+
|
|
314
|
+
Args:
|
|
315
|
+
after: Only snapshots bookmarked after this Unix time.
|
|
316
|
+
before: Only snapshots bookmarked before this Unix time.
|
|
317
|
+
filter_type: How ``filter_patterns`` match (default ``exact``).
|
|
318
|
+
filter_patterns: Patterns to match.
|
|
319
|
+
status: Only snapshots with this status.
|
|
320
|
+
url__icontains: URL contains.
|
|
321
|
+
url__istartswith: URL starts with.
|
|
322
|
+
tag: Only snapshots with this tag.
|
|
323
|
+
crawl_id: Only snapshots of this crawl.
|
|
324
|
+
limit: Maximum snapshots.
|
|
325
|
+
sort: Sort field.
|
|
326
|
+
search: Full-text search.
|
|
327
|
+
timeout: Seconds to wait for the removal.
|
|
328
|
+
"""
|
|
329
|
+
if not any(
|
|
330
|
+
(filter_patterns, url__icontains, url__istartswith, tag, crawl_id, search)
|
|
331
|
+
):
|
|
332
|
+
raise ParameterError(
|
|
333
|
+
"cli_remove needs a selection (filter_patterns, tag, url__icontains,"
|
|
334
|
+
" url__istartswith, crawl_id or search); it never removes everything"
|
|
335
|
+
)
|
|
336
|
+
body = _selection(
|
|
337
|
+
after=after,
|
|
338
|
+
before=before,
|
|
339
|
+
filter_type=filter_type,
|
|
340
|
+
filter_patterns=filter_patterns,
|
|
341
|
+
status=status,
|
|
342
|
+
url__icontains=url__icontains,
|
|
343
|
+
url__istartswith=url__istartswith,
|
|
344
|
+
tag=tag,
|
|
345
|
+
crawl_id=crawl_id,
|
|
346
|
+
limit=limit,
|
|
347
|
+
sort=sort,
|
|
348
|
+
search=search,
|
|
349
|
+
) | {"timeout": timeout}
|
|
350
|
+
return self._send("POST", f"{_CLI}/remove", json=body)
|