wildweb-mcp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wildweb-mcp might be problematic. Click here for more details.
- wildweb/__init__.py +3 -0
- wildweb/budget.py +345 -0
- wildweb/cli.py +178 -0
- wildweb/config.py +431 -0
- wildweb/extraction.py +67 -0
- wildweb/logging_setup.py +221 -0
- wildweb/models.py +147 -0
- wildweb/scraper/__init__.py +1 -0
- wildweb/scraper/base.py +163 -0
- wildweb/scraper/playwright_scraper.py +283 -0
- wildweb/scraper/registry.py +137 -0
- wildweb/search/__init__.py +1 -0
- wildweb/search/base.py +172 -0
- wildweb/search/duckduckgo_provider.py +236 -0
- wildweb/search/registry.py +139 -0
- wildweb/server.py +345 -0
- wildweb/tools.py +741 -0
- wildweb/user_agents.py +71 -0
- wildweb_mcp-0.1.0.dist-info/METADATA +429 -0
- wildweb_mcp-0.1.0.dist-info/RECORD +24 -0
- wildweb_mcp-0.1.0.dist-info/WHEEL +5 -0
- wildweb_mcp-0.1.0.dist-info/entry_points.txt +2 -0
- wildweb_mcp-0.1.0.dist-info/licenses/LICENSE +21 -0
- wildweb_mcp-0.1.0.dist-info/top_level.txt +1 -0
wildweb/__init__.py
ADDED
wildweb/budget.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Token budgeting, truncation, and the continuation store.
|
|
2
|
+
|
|
3
|
+
This module is the pure-logic layer behind the ``scrape`` / ``scrape_continue``
|
|
4
|
+
token-budget behavior (Requirements 3.4, 3.5, 3.6). It is deliberately
|
|
5
|
+
independent of the MCP tool layer: it returns plain result dataclasses that
|
|
6
|
+
``tools.py`` maps onto :class:`wildweb.models.ToolOutput`. It does not import
|
|
7
|
+
``ToolOutput`` and performs no I/O beyond consulting an injectable clock.
|
|
8
|
+
|
|
9
|
+
Design notes
|
|
10
|
+
------------
|
|
11
|
+
Token counting and slicing go through a single injectable
|
|
12
|
+
:class:`TokenCounter` abstraction so that budgeting is self-consistent between
|
|
13
|
+
runtime and tests. The default :class:`TiktokenCounter` uses a ``tiktoken``
|
|
14
|
+
encoding and operates at the token-id level: content is encoded to a list of
|
|
15
|
+
token ids, slices are taken over that list, and each slice is decoded back to
|
|
16
|
+
text.
|
|
17
|
+
|
|
18
|
+
This token-id slicing is what makes Property 6 hold: for ``tiktoken`` encodings
|
|
19
|
+
``decode(encode(C)) == C`` on normal text, and ``decode`` is concatenative over
|
|
20
|
+
a partition of the token-id list, so decoding consecutive token-id slices and
|
|
21
|
+
concatenating them reconstructs exactly the original content. Consequently the
|
|
22
|
+
continuation offset is a *token* offset into the encoded token list, and each
|
|
23
|
+
slice is ``decode(tokens[offset:offset + budget])``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import secrets
|
|
29
|
+
import time
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
from typing import Callable, Protocol, runtime_checkable
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# --------------------------------------------------------------------------- #
|
|
35
|
+
# Token counting / slicing abstraction
|
|
36
|
+
# --------------------------------------------------------------------------- #
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@runtime_checkable
|
|
40
|
+
class TokenCounter(Protocol):
|
|
41
|
+
"""Injectable token counting and slicing abstraction.
|
|
42
|
+
|
|
43
|
+
Implementations operate at the token-id level so budgeting can take an
|
|
44
|
+
exact "largest prefix <= budget" slice and reconstruct content by decoding
|
|
45
|
+
consecutive token-id slices (see module docstring, Property 6).
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def count(self, text: str) -> int:
|
|
49
|
+
"""Return the number of tokens in ``text``."""
|
|
50
|
+
...
|
|
51
|
+
|
|
52
|
+
def encode(self, text: str) -> list[int]:
|
|
53
|
+
"""Encode ``text`` into a list of token ids."""
|
|
54
|
+
...
|
|
55
|
+
|
|
56
|
+
def decode(self, tokens: list[int]) -> str:
|
|
57
|
+
"""Decode a list of token ids back into text."""
|
|
58
|
+
...
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class TiktokenCounter:
|
|
62
|
+
"""Default :class:`TokenCounter` backed by a ``tiktoken`` encoding.
|
|
63
|
+
|
|
64
|
+
The encoding name is injectable (default ``cl100k_base``). An explicit
|
|
65
|
+
encoder object may also be supplied, primarily for testing.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
encoding_name: str = "cl100k_base",
|
|
71
|
+
encoder: object | None = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Create a counter for the given ``tiktoken`` encoding.
|
|
74
|
+
|
|
75
|
+
Args:
|
|
76
|
+
encoding_name: Name passed to ``tiktoken.get_encoding`` when no
|
|
77
|
+
explicit ``encoder`` is supplied.
|
|
78
|
+
encoder: An optional pre-built encoder exposing ``encode`` /
|
|
79
|
+
``decode``; when provided, ``encoding_name`` is ignored.
|
|
80
|
+
"""
|
|
81
|
+
if encoder is None:
|
|
82
|
+
import tiktoken
|
|
83
|
+
|
|
84
|
+
encoder = tiktoken.get_encoding(encoding_name)
|
|
85
|
+
self._encoder = encoder
|
|
86
|
+
self._encoding_name = encoding_name
|
|
87
|
+
|
|
88
|
+
def count(self, text: str) -> int:
|
|
89
|
+
return len(self._encoder.encode(text))
|
|
90
|
+
|
|
91
|
+
def encode(self, text: str) -> list[int]:
|
|
92
|
+
return list(self._encoder.encode(text))
|
|
93
|
+
|
|
94
|
+
def decode(self, tokens: list[int]) -> str:
|
|
95
|
+
return self._encoder.decode(tokens)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# --------------------------------------------------------------------------- #
|
|
99
|
+
# Continuation store
|
|
100
|
+
# --------------------------------------------------------------------------- #
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass
|
|
104
|
+
class ContinuationEntry:
|
|
105
|
+
"""A cached continuation record stored under an opaque token.
|
|
106
|
+
|
|
107
|
+
Attributes:
|
|
108
|
+
full_content: The complete extracted content for the originating scrape.
|
|
109
|
+
next_offset: Token offset of the next slice to return (an index into
|
|
110
|
+
the ``full_content`` encoded token list).
|
|
111
|
+
created_at: Epoch timestamp (seconds) when the entry was stored, used
|
|
112
|
+
for TTL expiry.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
full_content: str
|
|
116
|
+
next_offset: int
|
|
117
|
+
created_at: float
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
class ContinuationStore:
|
|
121
|
+
"""In-memory store mapping opaque tokens to :class:`ContinuationEntry`.
|
|
122
|
+
|
|
123
|
+
Tokens are cryptographically random and opaque: they carry no
|
|
124
|
+
client-trusted data (generated via :func:`secrets.token_urlsafe`). Entries
|
|
125
|
+
older than ``ttl_seconds`` are treated as expired and purged on access
|
|
126
|
+
(Requirement 3.6).
|
|
127
|
+
|
|
128
|
+
The clock is injectable for testing. ``time.time`` (epoch seconds) is used
|
|
129
|
+
by default rather than ``time.monotonic`` so that ``created_at`` compares
|
|
130
|
+
directly against a human-meaningful TTL and survives being reasoned about in
|
|
131
|
+
tests; the store only ever compares two readings from the same clock.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
ttl_seconds: float,
|
|
137
|
+
clock: Callable[[], float] = time.time,
|
|
138
|
+
token_factory: Callable[[], str] | None = None,
|
|
139
|
+
) -> None:
|
|
140
|
+
"""Create a store with the given TTL and injectable clock/token source.
|
|
141
|
+
|
|
142
|
+
Args:
|
|
143
|
+
ttl_seconds: Lifetime of an entry in seconds; a lookup strictly
|
|
144
|
+
after ``created_at + ttl_seconds`` is treated as expired.
|
|
145
|
+
clock: Zero-argument callable returning the current time in seconds.
|
|
146
|
+
token_factory: Zero-argument callable producing a fresh opaque
|
|
147
|
+
token; defaults to a 32-byte URL-safe random token.
|
|
148
|
+
"""
|
|
149
|
+
self._ttl_seconds = ttl_seconds
|
|
150
|
+
self._clock = clock
|
|
151
|
+
self._token_factory = token_factory or (lambda: secrets.token_urlsafe(32))
|
|
152
|
+
self._entries: dict[str, ContinuationEntry] = {}
|
|
153
|
+
|
|
154
|
+
def put(self, entry: ContinuationEntry) -> str:
|
|
155
|
+
"""Store ``entry`` under a fresh opaque token and return the token."""
|
|
156
|
+
token = self._token_factory()
|
|
157
|
+
# Guard against the astronomically unlikely token collision.
|
|
158
|
+
while token in self._entries:
|
|
159
|
+
token = self._token_factory()
|
|
160
|
+
self._entries[token] = entry
|
|
161
|
+
return token
|
|
162
|
+
|
|
163
|
+
def get(self, token: str) -> ContinuationEntry | None:
|
|
164
|
+
"""Return the entry for ``token`` or ``None`` if absent or expired.
|
|
165
|
+
|
|
166
|
+
Expired entries are purged as a side effect.
|
|
167
|
+
"""
|
|
168
|
+
entry = self._entries.get(token)
|
|
169
|
+
if entry is None:
|
|
170
|
+
return None
|
|
171
|
+
if self._is_expired(entry):
|
|
172
|
+
self._entries.pop(token, None)
|
|
173
|
+
return None
|
|
174
|
+
return entry
|
|
175
|
+
|
|
176
|
+
def delete(self, token: str) -> None:
|
|
177
|
+
"""Remove ``token`` from the store if present (idempotent)."""
|
|
178
|
+
self._entries.pop(token, None)
|
|
179
|
+
|
|
180
|
+
def now(self) -> float:
|
|
181
|
+
"""Return the store's current clock reading (seconds).
|
|
182
|
+
|
|
183
|
+
Exposed so entries created by the budgeting functions share the same
|
|
184
|
+
time source the store uses for TTL comparisons.
|
|
185
|
+
"""
|
|
186
|
+
return self._clock()
|
|
187
|
+
|
|
188
|
+
def _is_expired(self, entry: ContinuationEntry) -> bool:
|
|
189
|
+
return self._clock() > entry.created_at + self._ttl_seconds
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# --------------------------------------------------------------------------- #
|
|
193
|
+
# Budgeting results
|
|
194
|
+
# --------------------------------------------------------------------------- #
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass(frozen=True)
|
|
198
|
+
class BudgetResult:
|
|
199
|
+
"""Result of applying a token budget to freshly extracted content.
|
|
200
|
+
|
|
201
|
+
Attributes:
|
|
202
|
+
text: The slice of content to return in this response.
|
|
203
|
+
truncated: ``True`` when the full content exceeded the budget and more
|
|
204
|
+
content remains behind ``continuation``.
|
|
205
|
+
continuation: A fresh opaque continuation token when ``truncated``;
|
|
206
|
+
``None`` otherwise.
|
|
207
|
+
"""
|
|
208
|
+
|
|
209
|
+
text: str
|
|
210
|
+
truncated: bool
|
|
211
|
+
continuation: str | None
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
@dataclass(frozen=True)
|
|
215
|
+
class ContinueResult:
|
|
216
|
+
"""Result of resolving a continuation token.
|
|
217
|
+
|
|
218
|
+
Attributes:
|
|
219
|
+
invalid: ``True`` when the token was absent or expired; the tool layer
|
|
220
|
+
maps this to an ``invalid_continuation`` error with empty content.
|
|
221
|
+
text: The next slice of content; always ``""`` when ``invalid``.
|
|
222
|
+
truncated: ``True`` when more content remains behind ``continuation``.
|
|
223
|
+
continuation: A fresh opaque continuation token when more content
|
|
224
|
+
remains; ``None`` on the final slice or when ``invalid``.
|
|
225
|
+
"""
|
|
226
|
+
|
|
227
|
+
invalid: bool
|
|
228
|
+
text: str
|
|
229
|
+
truncated: bool
|
|
230
|
+
continuation: str | None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# --------------------------------------------------------------------------- #
|
|
234
|
+
# Budgeting functions
|
|
235
|
+
# --------------------------------------------------------------------------- #
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def apply_budget(
|
|
239
|
+
content: str,
|
|
240
|
+
token_budget: int | None,
|
|
241
|
+
counter: TokenCounter,
|
|
242
|
+
store: ContinuationStore,
|
|
243
|
+
) -> BudgetResult:
|
|
244
|
+
"""Apply ``token_budget`` to freshly extracted ``content``.
|
|
245
|
+
|
|
246
|
+
When budgeting is disabled (``token_budget is None``) or the content fits
|
|
247
|
+
within the budget, the full content is returned untruncated with no
|
|
248
|
+
continuation. Otherwise the largest token prefix ``<= token_budget`` is
|
|
249
|
+
returned, ``truncated`` is set, and a :class:`ContinuationEntry` holding the
|
|
250
|
+
full content and the next token offset is stored under a fresh token
|
|
251
|
+
(Requirements 3.4, 3.5).
|
|
252
|
+
|
|
253
|
+
Args:
|
|
254
|
+
content: The complete extracted content.
|
|
255
|
+
token_budget: Maximum tokens for this response, or ``None`` to disable.
|
|
256
|
+
counter: The token counter / slicer used for encoding and decoding.
|
|
257
|
+
store: The continuation store used to cache the remainder.
|
|
258
|
+
|
|
259
|
+
Returns:
|
|
260
|
+
A :class:`BudgetResult` describing the slice to return.
|
|
261
|
+
"""
|
|
262
|
+
if token_budget is None or token_budget < 0:
|
|
263
|
+
return BudgetResult(text=content, truncated=False, continuation=None)
|
|
264
|
+
|
|
265
|
+
tokens = counter.encode(content)
|
|
266
|
+
if len(tokens) <= token_budget:
|
|
267
|
+
return BudgetResult(text=content, truncated=False, continuation=None)
|
|
268
|
+
|
|
269
|
+
prefix = counter.decode(tokens[:token_budget])
|
|
270
|
+
entry = ContinuationEntry(
|
|
271
|
+
full_content=content,
|
|
272
|
+
next_offset=token_budget,
|
|
273
|
+
created_at=store.now(),
|
|
274
|
+
)
|
|
275
|
+
token = store.put(entry)
|
|
276
|
+
return BudgetResult(text=prefix, truncated=True, continuation=token)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def continue_budget(
|
|
280
|
+
token: str,
|
|
281
|
+
token_budget: int | None,
|
|
282
|
+
counter: TokenCounter,
|
|
283
|
+
store: ContinuationStore,
|
|
284
|
+
) -> ContinueResult:
|
|
285
|
+
"""Resolve a continuation ``token`` and return the next slice.
|
|
286
|
+
|
|
287
|
+
If the token is absent or expired the result is flagged ``invalid`` with
|
|
288
|
+
empty content (the tool layer maps this to an ``invalid_continuation``
|
|
289
|
+
error, Requirement 3.6). Otherwise the next ``token_budget``-sized token
|
|
290
|
+
slice is decoded and returned; when content still remains a fresh
|
|
291
|
+
continuation token is issued (and the old token retired), and on the final
|
|
292
|
+
slice no continuation is issued and ``truncated`` is ``False``
|
|
293
|
+
(Requirement 3.5).
|
|
294
|
+
|
|
295
|
+
When ``token_budget is None`` the entire remaining content is returned as a
|
|
296
|
+
single final slice.
|
|
297
|
+
|
|
298
|
+
Args:
|
|
299
|
+
token: The opaque continuation token from a prior truncated result.
|
|
300
|
+
token_budget: Maximum tokens for this response, or ``None`` to return
|
|
301
|
+
all remaining content at once.
|
|
302
|
+
counter: The token counter / slicer used for encoding and decoding.
|
|
303
|
+
store: The continuation store holding the cached content.
|
|
304
|
+
|
|
305
|
+
Returns:
|
|
306
|
+
A :class:`ContinueResult` describing the next slice, or an invalid
|
|
307
|
+
result when the token cannot be resolved.
|
|
308
|
+
"""
|
|
309
|
+
entry = store.get(token)
|
|
310
|
+
if entry is None:
|
|
311
|
+
return ContinueResult(
|
|
312
|
+
invalid=True, text="", truncated=False, continuation=None
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
tokens = counter.encode(entry.full_content)
|
|
316
|
+
start = entry.next_offset
|
|
317
|
+
|
|
318
|
+
if token_budget is None or token_budget < 0:
|
|
319
|
+
end = len(tokens)
|
|
320
|
+
else:
|
|
321
|
+
end = min(start + token_budget, len(tokens))
|
|
322
|
+
|
|
323
|
+
slice_text = counter.decode(tokens[start:end])
|
|
324
|
+
|
|
325
|
+
# Retire the token being consumed; a fresh token is issued below if needed.
|
|
326
|
+
store.delete(token)
|
|
327
|
+
|
|
328
|
+
if end >= len(tokens):
|
|
329
|
+
# Final slice: no more content remains.
|
|
330
|
+
return ContinueResult(
|
|
331
|
+
invalid=False, text=slice_text, truncated=False, continuation=None
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
next_entry = ContinuationEntry(
|
|
335
|
+
full_content=entry.full_content,
|
|
336
|
+
next_offset=end,
|
|
337
|
+
created_at=store.now(),
|
|
338
|
+
)
|
|
339
|
+
next_token = store.put(next_entry)
|
|
340
|
+
return ContinueResult(
|
|
341
|
+
invalid=False,
|
|
342
|
+
text=slice_text,
|
|
343
|
+
truncated=True,
|
|
344
|
+
continuation=next_token,
|
|
345
|
+
)
|
wildweb/cli.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Command-line entry point and config-path resolution for the wildweb server.
|
|
2
|
+
|
|
3
|
+
This module is the console entry point declared in ``pyproject.toml``
|
|
4
|
+
(``wildweb = "wildweb.cli:main"``). It wires together configuration
|
|
5
|
+
loading, logging setup, and the async MCP server (the composition root).
|
|
6
|
+
|
|
7
|
+
Config path precedence (Requirements 6.4-6.6):
|
|
8
|
+
|
|
9
|
+
1. ``--config <path>`` CLI argument (highest precedence).
|
|
10
|
+
2. ``WILDWEB_CONFIG`` environment variable (used only when the CLI arg is
|
|
11
|
+
absent or empty).
|
|
12
|
+
3. If neither is present, no config file is loaded at all: the server runs
|
|
13
|
+
with the built-in :class:`~wildweb.config.Config` defaults.
|
|
14
|
+
|
|
15
|
+
The precedence logic lives in :func:`resolve_config_path`, which is pure and
|
|
16
|
+
side-effect free (no ``sys.exit``, no stdout/stderr) so it can be exercised
|
|
17
|
+
directly by property tests. :func:`main` is the thin, side-effecting shell that
|
|
18
|
+
parses args, resolves the path, loads config, configures logging, and hands off
|
|
19
|
+
to the composition root via :func:`run`.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import os
|
|
26
|
+
import sys
|
|
27
|
+
from typing import Mapping
|
|
28
|
+
|
|
29
|
+
from wildweb.config import Config, ConfigError, ConfigLoader
|
|
30
|
+
from wildweb.logging_setup import configure_logging
|
|
31
|
+
|
|
32
|
+
#: Environment variable consulted when no ``--config`` CLI argument is supplied.
|
|
33
|
+
ENV_CONFIG_VAR = "WILDWEB_CONFIG"
|
|
34
|
+
|
|
35
|
+
#: Exit code used for config-load failures at startup.
|
|
36
|
+
EXIT_CONFIG_ERROR = 2
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def resolve_config_path(
|
|
40
|
+
cli_config: str | None,
|
|
41
|
+
env: Mapping[str, str] | None = None,
|
|
42
|
+
) -> str | None:
|
|
43
|
+
"""Resolve the configuration file path by precedence.
|
|
44
|
+
|
|
45
|
+
Precedence (Requirements 6.4-6.6):
|
|
46
|
+
|
|
47
|
+
1. ``cli_config`` when present and non-empty (highest).
|
|
48
|
+
2. ``env[WILDWEB_CONFIG]`` when present and non-empty.
|
|
49
|
+
3. Otherwise ``None``, meaning: no config file was supplied, so the caller
|
|
50
|
+
should fall back to the built-in :class:`~wildweb.config.Config`
|
|
51
|
+
defaults instead of loading a file.
|
|
52
|
+
|
|
53
|
+
This function is pure and side-effect free: it performs no I/O, never calls
|
|
54
|
+
``sys.exit``, and writes nothing to stdout/stderr. It is the unit targeted
|
|
55
|
+
by the path-precedence property test.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
cli_config: The value of the ``--config`` CLI argument, or ``None`` when
|
|
59
|
+
the flag was not supplied. Empty/whitespace-only values are treated
|
|
60
|
+
as absent.
|
|
61
|
+
env: Mapping consulted for ``WILDWEB_CONFIG``. Defaults to
|
|
62
|
+
:data:`os.environ` when ``None``.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
The resolved config file path, or ``None`` when neither source
|
|
66
|
+
supplies a non-empty path.
|
|
67
|
+
"""
|
|
68
|
+
if cli_config is not None and cli_config.strip():
|
|
69
|
+
return cli_config
|
|
70
|
+
|
|
71
|
+
if env is None:
|
|
72
|
+
env = os.environ
|
|
73
|
+
|
|
74
|
+
env_value = env.get(ENV_CONFIG_VAR)
|
|
75
|
+
if env_value is not None and env_value.strip():
|
|
76
|
+
return env_value
|
|
77
|
+
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
82
|
+
"""Parse command-line arguments.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
argv: Argument list excluding the program name. Defaults to
|
|
86
|
+
``sys.argv[1:]`` when ``None``.
|
|
87
|
+
|
|
88
|
+
Returns:
|
|
89
|
+
A namespace with a ``config`` attribute (``str | None``) holding the
|
|
90
|
+
``--config`` value, or ``None`` when the flag was omitted.
|
|
91
|
+
"""
|
|
92
|
+
parser = argparse.ArgumentParser(
|
|
93
|
+
prog="wildweb",
|
|
94
|
+
description=(
|
|
95
|
+
"wildweb MCP server: exposes web scraping as an MCP tool over "
|
|
96
|
+
"stdio."
|
|
97
|
+
),
|
|
98
|
+
)
|
|
99
|
+
parser.add_argument(
|
|
100
|
+
"--config",
|
|
101
|
+
dest="config",
|
|
102
|
+
default=None,
|
|
103
|
+
metavar="PATH",
|
|
104
|
+
help=(
|
|
105
|
+
"path to the JSON configuration file; overrides the "
|
|
106
|
+
f"{ENV_CONFIG_VAR} environment variable"
|
|
107
|
+
),
|
|
108
|
+
)
|
|
109
|
+
return parser.parse_args(argv)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def run(config: Config) -> int:
|
|
113
|
+
"""Hand off to the composition root and run the async MCP server.
|
|
114
|
+
|
|
115
|
+
This is the seam between the CLI and the server. The server module is
|
|
116
|
+
imported lazily so that importing :mod:`wildweb.cli` (and calling
|
|
117
|
+
:func:`resolve_config_path` / :func:`parse_args`) does not require the
|
|
118
|
+
server/composition root to exist yet. The composition root
|
|
119
|
+
(``wildweb.server.run_server``) builds the registry, registers the
|
|
120
|
+
Playwright scraper, starts it, serves over stdio, and releases resources on
|
|
121
|
+
shutdown.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
config: The validated configuration driving the server.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
The process exit code (``0`` on a clean server shutdown).
|
|
128
|
+
"""
|
|
129
|
+
import asyncio
|
|
130
|
+
|
|
131
|
+
from wildweb.server import run_server
|
|
132
|
+
|
|
133
|
+
return asyncio.run(run_server(config))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def main(argv: list[str] | None = None) -> int:
|
|
137
|
+
"""CLI entry point.
|
|
138
|
+
|
|
139
|
+
Steps:
|
|
140
|
+
|
|
141
|
+
1. Parse arguments (``argv`` defaults to ``sys.argv[1:]``).
|
|
142
|
+
2. Resolve the config path by precedence (Requirements 6.4-6.6). When
|
|
143
|
+
neither the ``--config`` argument nor the ``WILDWEB_CONFIG`` environment
|
|
144
|
+
variable is supplied, fall back to the built-in :class:`Config`
|
|
145
|
+
defaults rather than failing.
|
|
146
|
+
3. Otherwise, load and validate the config file (Requirements 5.3-5.5,
|
|
147
|
+
7.5). On :class:`ConfigError`, write the message to stderr and return
|
|
148
|
+
non-zero.
|
|
149
|
+
4. Configure logging from the resolved log level (Requirements 7.3, 7.4).
|
|
150
|
+
5. Hand off to the composition root via :func:`run`.
|
|
151
|
+
|
|
152
|
+
Args:
|
|
153
|
+
argv: Argument list excluding the program name. Defaults to
|
|
154
|
+
``sys.argv[1:]`` when ``None``.
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
The process exit code. Used directly as the exit status by the
|
|
158
|
+
``console_scripts`` entry point.
|
|
159
|
+
"""
|
|
160
|
+
args = parse_args(argv)
|
|
161
|
+
config_path = resolve_config_path(args.config, os.environ)
|
|
162
|
+
|
|
163
|
+
if config_path is None:
|
|
164
|
+
config = Config()
|
|
165
|
+
else:
|
|
166
|
+
try:
|
|
167
|
+
config = ConfigLoader().load(config_path)
|
|
168
|
+
except ConfigError as exc:
|
|
169
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
170
|
+
return EXIT_CONFIG_ERROR
|
|
171
|
+
|
|
172
|
+
configure_logging(config)
|
|
173
|
+
|
|
174
|
+
return run(config)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
if __name__ == "__main__":
|
|
178
|
+
sys.exit(main())
|