wildweb-mcp 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of wildweb-mcp might be problematic. Click here for more details.

wildweb/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """wildweb: an extensible MCP server that exposes web scraping as an MCP tool."""
2
+
3
+ __version__ = "0.1.0"
wildweb/budget.py ADDED
@@ -0,0 +1,345 @@
1
+ """Token budgeting, truncation, and the continuation store.
2
+
3
+ This module is the pure-logic layer behind the ``scrape`` / ``scrape_continue``
4
+ token-budget behavior (Requirements 3.4, 3.5, 3.6). It is deliberately
5
+ independent of the MCP tool layer: it returns plain result dataclasses that
6
+ ``tools.py`` maps onto :class:`wildweb.models.ToolOutput`. It does not import
7
+ ``ToolOutput`` and performs no I/O beyond consulting an injectable clock.
8
+
9
+ Design notes
10
+ ------------
11
+ Token counting and slicing go through a single injectable
12
+ :class:`TokenCounter` abstraction so that budgeting is self-consistent between
13
+ runtime and tests. The default :class:`TiktokenCounter` uses a ``tiktoken``
14
+ encoding and operates at the token-id level: content is encoded to a list of
15
+ token ids, slices are taken over that list, and each slice is decoded back to
16
+ text.
17
+
18
+ This token-id slicing is what makes Property 6 hold: for ``tiktoken`` encodings
19
+ ``decode(encode(C)) == C`` on normal text, and ``decode`` is concatenative over
20
+ a partition of the token-id list, so decoding consecutive token-id slices and
21
+ concatenating them reconstructs exactly the original content. Consequently the
22
+ continuation offset is a *token* offset into the encoded token list, and each
23
+ slice is ``decode(tokens[offset:offset + budget])``.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import secrets
29
+ import time
30
+ from dataclasses import dataclass
31
+ from typing import Callable, Protocol, runtime_checkable
32
+
33
+
34
+ # --------------------------------------------------------------------------- #
35
+ # Token counting / slicing abstraction
36
+ # --------------------------------------------------------------------------- #
37
+
38
+
39
+ @runtime_checkable
40
+ class TokenCounter(Protocol):
41
+ """Injectable token counting and slicing abstraction.
42
+
43
+ Implementations operate at the token-id level so budgeting can take an
44
+ exact "largest prefix <= budget" slice and reconstruct content by decoding
45
+ consecutive token-id slices (see module docstring, Property 6).
46
+ """
47
+
48
+ def count(self, text: str) -> int:
49
+ """Return the number of tokens in ``text``."""
50
+ ...
51
+
52
+ def encode(self, text: str) -> list[int]:
53
+ """Encode ``text`` into a list of token ids."""
54
+ ...
55
+
56
+ def decode(self, tokens: list[int]) -> str:
57
+ """Decode a list of token ids back into text."""
58
+ ...
59
+
60
+
61
+ class TiktokenCounter:
62
+ """Default :class:`TokenCounter` backed by a ``tiktoken`` encoding.
63
+
64
+ The encoding name is injectable (default ``cl100k_base``). An explicit
65
+ encoder object may also be supplied, primarily for testing.
66
+ """
67
+
68
+ def __init__(
69
+ self,
70
+ encoding_name: str = "cl100k_base",
71
+ encoder: object | None = None,
72
+ ) -> None:
73
+ """Create a counter for the given ``tiktoken`` encoding.
74
+
75
+ Args:
76
+ encoding_name: Name passed to ``tiktoken.get_encoding`` when no
77
+ explicit ``encoder`` is supplied.
78
+ encoder: An optional pre-built encoder exposing ``encode`` /
79
+ ``decode``; when provided, ``encoding_name`` is ignored.
80
+ """
81
+ if encoder is None:
82
+ import tiktoken
83
+
84
+ encoder = tiktoken.get_encoding(encoding_name)
85
+ self._encoder = encoder
86
+ self._encoding_name = encoding_name
87
+
88
+ def count(self, text: str) -> int:
89
+ return len(self._encoder.encode(text))
90
+
91
+ def encode(self, text: str) -> list[int]:
92
+ return list(self._encoder.encode(text))
93
+
94
+ def decode(self, tokens: list[int]) -> str:
95
+ return self._encoder.decode(tokens)
96
+
97
+
98
+ # --------------------------------------------------------------------------- #
99
+ # Continuation store
100
+ # --------------------------------------------------------------------------- #
101
+
102
+
103
+ @dataclass
104
+ class ContinuationEntry:
105
+ """A cached continuation record stored under an opaque token.
106
+
107
+ Attributes:
108
+ full_content: The complete extracted content for the originating scrape.
109
+ next_offset: Token offset of the next slice to return (an index into
110
+ the ``full_content`` encoded token list).
111
+ created_at: Epoch timestamp (seconds) when the entry was stored, used
112
+ for TTL expiry.
113
+ """
114
+
115
+ full_content: str
116
+ next_offset: int
117
+ created_at: float
118
+
119
+
120
+ class ContinuationStore:
121
+ """In-memory store mapping opaque tokens to :class:`ContinuationEntry`.
122
+
123
+ Tokens are cryptographically random and opaque: they carry no
124
+ client-trusted data (generated via :func:`secrets.token_urlsafe`). Entries
125
+ older than ``ttl_seconds`` are treated as expired and purged on access
126
+ (Requirement 3.6).
127
+
128
+ The clock is injectable for testing. ``time.time`` (epoch seconds) is used
129
+ by default rather than ``time.monotonic`` so that ``created_at`` compares
130
+ directly against a human-meaningful TTL and survives being reasoned about in
131
+ tests; the store only ever compares two readings from the same clock.
132
+ """
133
+
134
+ def __init__(
135
+ self,
136
+ ttl_seconds: float,
137
+ clock: Callable[[], float] = time.time,
138
+ token_factory: Callable[[], str] | None = None,
139
+ ) -> None:
140
+ """Create a store with the given TTL and injectable clock/token source.
141
+
142
+ Args:
143
+ ttl_seconds: Lifetime of an entry in seconds; a lookup strictly
144
+ after ``created_at + ttl_seconds`` is treated as expired.
145
+ clock: Zero-argument callable returning the current time in seconds.
146
+ token_factory: Zero-argument callable producing a fresh opaque
147
+ token; defaults to a 32-byte URL-safe random token.
148
+ """
149
+ self._ttl_seconds = ttl_seconds
150
+ self._clock = clock
151
+ self._token_factory = token_factory or (lambda: secrets.token_urlsafe(32))
152
+ self._entries: dict[str, ContinuationEntry] = {}
153
+
154
+ def put(self, entry: ContinuationEntry) -> str:
155
+ """Store ``entry`` under a fresh opaque token and return the token."""
156
+ token = self._token_factory()
157
+ # Guard against the astronomically unlikely token collision.
158
+ while token in self._entries:
159
+ token = self._token_factory()
160
+ self._entries[token] = entry
161
+ return token
162
+
163
+ def get(self, token: str) -> ContinuationEntry | None:
164
+ """Return the entry for ``token`` or ``None`` if absent or expired.
165
+
166
+ Expired entries are purged as a side effect.
167
+ """
168
+ entry = self._entries.get(token)
169
+ if entry is None:
170
+ return None
171
+ if self._is_expired(entry):
172
+ self._entries.pop(token, None)
173
+ return None
174
+ return entry
175
+
176
+ def delete(self, token: str) -> None:
177
+ """Remove ``token`` from the store if present (idempotent)."""
178
+ self._entries.pop(token, None)
179
+
180
+ def now(self) -> float:
181
+ """Return the store's current clock reading (seconds).
182
+
183
+ Exposed so entries created by the budgeting functions share the same
184
+ time source the store uses for TTL comparisons.
185
+ """
186
+ return self._clock()
187
+
188
+ def _is_expired(self, entry: ContinuationEntry) -> bool:
189
+ return self._clock() > entry.created_at + self._ttl_seconds
190
+
191
+
192
+ # --------------------------------------------------------------------------- #
193
+ # Budgeting results
194
+ # --------------------------------------------------------------------------- #
195
+
196
+
197
+ @dataclass(frozen=True)
198
+ class BudgetResult:
199
+ """Result of applying a token budget to freshly extracted content.
200
+
201
+ Attributes:
202
+ text: The slice of content to return in this response.
203
+ truncated: ``True`` when the full content exceeded the budget and more
204
+ content remains behind ``continuation``.
205
+ continuation: A fresh opaque continuation token when ``truncated``;
206
+ ``None`` otherwise.
207
+ """
208
+
209
+ text: str
210
+ truncated: bool
211
+ continuation: str | None
212
+
213
+
214
+ @dataclass(frozen=True)
215
+ class ContinueResult:
216
+ """Result of resolving a continuation token.
217
+
218
+ Attributes:
219
+ invalid: ``True`` when the token was absent or expired; the tool layer
220
+ maps this to an ``invalid_continuation`` error with empty content.
221
+ text: The next slice of content; always ``""`` when ``invalid``.
222
+ truncated: ``True`` when more content remains behind ``continuation``.
223
+ continuation: A fresh opaque continuation token when more content
224
+ remains; ``None`` on the final slice or when ``invalid``.
225
+ """
226
+
227
+ invalid: bool
228
+ text: str
229
+ truncated: bool
230
+ continuation: str | None
231
+
232
+
233
+ # --------------------------------------------------------------------------- #
234
+ # Budgeting functions
235
+ # --------------------------------------------------------------------------- #
236
+
237
+
238
+ def apply_budget(
239
+ content: str,
240
+ token_budget: int | None,
241
+ counter: TokenCounter,
242
+ store: ContinuationStore,
243
+ ) -> BudgetResult:
244
+ """Apply ``token_budget`` to freshly extracted ``content``.
245
+
246
+ When budgeting is disabled (``token_budget is None``) or the content fits
247
+ within the budget, the full content is returned untruncated with no
248
+ continuation. Otherwise the largest token prefix ``<= token_budget`` is
249
+ returned, ``truncated`` is set, and a :class:`ContinuationEntry` holding the
250
+ full content and the next token offset is stored under a fresh token
251
+ (Requirements 3.4, 3.5).
252
+
253
+ Args:
254
+ content: The complete extracted content.
255
+ token_budget: Maximum tokens for this response, or ``None`` to disable.
256
+ counter: The token counter / slicer used for encoding and decoding.
257
+ store: The continuation store used to cache the remainder.
258
+
259
+ Returns:
260
+ A :class:`BudgetResult` describing the slice to return.
261
+ """
262
+ if token_budget is None or token_budget < 0:
263
+ return BudgetResult(text=content, truncated=False, continuation=None)
264
+
265
+ tokens = counter.encode(content)
266
+ if len(tokens) <= token_budget:
267
+ return BudgetResult(text=content, truncated=False, continuation=None)
268
+
269
+ prefix = counter.decode(tokens[:token_budget])
270
+ entry = ContinuationEntry(
271
+ full_content=content,
272
+ next_offset=token_budget,
273
+ created_at=store.now(),
274
+ )
275
+ token = store.put(entry)
276
+ return BudgetResult(text=prefix, truncated=True, continuation=token)
277
+
278
+
279
+ def continue_budget(
280
+ token: str,
281
+ token_budget: int | None,
282
+ counter: TokenCounter,
283
+ store: ContinuationStore,
284
+ ) -> ContinueResult:
285
+ """Resolve a continuation ``token`` and return the next slice.
286
+
287
+ If the token is absent or expired the result is flagged ``invalid`` with
288
+ empty content (the tool layer maps this to an ``invalid_continuation``
289
+ error, Requirement 3.6). Otherwise the next ``token_budget``-sized token
290
+ slice is decoded and returned; when content still remains a fresh
291
+ continuation token is issued (and the old token retired), and on the final
292
+ slice no continuation is issued and ``truncated`` is ``False``
293
+ (Requirement 3.5).
294
+
295
+ When ``token_budget is None`` the entire remaining content is returned as a
296
+ single final slice.
297
+
298
+ Args:
299
+ token: The opaque continuation token from a prior truncated result.
300
+ token_budget: Maximum tokens for this response, or ``None`` to return
301
+ all remaining content at once.
302
+ counter: The token counter / slicer used for encoding and decoding.
303
+ store: The continuation store holding the cached content.
304
+
305
+ Returns:
306
+ A :class:`ContinueResult` describing the next slice, or an invalid
307
+ result when the token cannot be resolved.
308
+ """
309
+ entry = store.get(token)
310
+ if entry is None:
311
+ return ContinueResult(
312
+ invalid=True, text="", truncated=False, continuation=None
313
+ )
314
+
315
+ tokens = counter.encode(entry.full_content)
316
+ start = entry.next_offset
317
+
318
+ if token_budget is None or token_budget < 0:
319
+ end = len(tokens)
320
+ else:
321
+ end = min(start + token_budget, len(tokens))
322
+
323
+ slice_text = counter.decode(tokens[start:end])
324
+
325
+ # Retire the token being consumed; a fresh token is issued below if needed.
326
+ store.delete(token)
327
+
328
+ if end >= len(tokens):
329
+ # Final slice: no more content remains.
330
+ return ContinueResult(
331
+ invalid=False, text=slice_text, truncated=False, continuation=None
332
+ )
333
+
334
+ next_entry = ContinuationEntry(
335
+ full_content=entry.full_content,
336
+ next_offset=end,
337
+ created_at=store.now(),
338
+ )
339
+ next_token = store.put(next_entry)
340
+ return ContinueResult(
341
+ invalid=False,
342
+ text=slice_text,
343
+ truncated=True,
344
+ continuation=next_token,
345
+ )
wildweb/cli.py ADDED
@@ -0,0 +1,178 @@
1
+ """Command-line entry point and config-path resolution for the wildweb server.
2
+
3
+ This module is the console entry point declared in ``pyproject.toml``
4
+ (``wildweb = "wildweb.cli:main"``). It wires together configuration
5
+ loading, logging setup, and the async MCP server (the composition root).
6
+
7
+ Config path precedence (Requirements 6.4-6.6):
8
+
9
+ 1. ``--config <path>`` CLI argument (highest precedence).
10
+ 2. ``WILDWEB_CONFIG`` environment variable (used only when the CLI arg is
11
+ absent or empty).
12
+ 3. If neither is present, no config file is loaded at all: the server runs
13
+ with the built-in :class:`~wildweb.config.Config` defaults.
14
+
15
+ The precedence logic lives in :func:`resolve_config_path`, which is pure and
16
+ side-effect free (no ``sys.exit``, no stdout/stderr) so it can be exercised
17
+ directly by property tests. :func:`main` is the thin, side-effecting shell that
18
+ parses args, resolves the path, loads config, configures logging, and hands off
19
+ to the composition root via :func:`run`.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import argparse
25
+ import os
26
+ import sys
27
+ from typing import Mapping
28
+
29
+ from wildweb.config import Config, ConfigError, ConfigLoader
30
+ from wildweb.logging_setup import configure_logging
31
+
32
+ #: Environment variable consulted when no ``--config`` CLI argument is supplied.
33
+ ENV_CONFIG_VAR = "WILDWEB_CONFIG"
34
+
35
+ #: Exit code used for config-load failures at startup.
36
+ EXIT_CONFIG_ERROR = 2
37
+
38
+
39
+ def resolve_config_path(
40
+ cli_config: str | None,
41
+ env: Mapping[str, str] | None = None,
42
+ ) -> str | None:
43
+ """Resolve the configuration file path by precedence.
44
+
45
+ Precedence (Requirements 6.4-6.6):
46
+
47
+ 1. ``cli_config`` when present and non-empty (highest).
48
+ 2. ``env[WILDWEB_CONFIG]`` when present and non-empty.
49
+ 3. Otherwise ``None``, meaning: no config file was supplied, so the caller
50
+ should fall back to the built-in :class:`~wildweb.config.Config`
51
+ defaults instead of loading a file.
52
+
53
+ This function is pure and side-effect free: it performs no I/O, never calls
54
+ ``sys.exit``, and writes nothing to stdout/stderr. It is the unit targeted
55
+ by the path-precedence property test.
56
+
57
+ Args:
58
+ cli_config: The value of the ``--config`` CLI argument, or ``None`` when
59
+ the flag was not supplied. Empty/whitespace-only values are treated
60
+ as absent.
61
+ env: Mapping consulted for ``WILDWEB_CONFIG``. Defaults to
62
+ :data:`os.environ` when ``None``.
63
+
64
+ Returns:
65
+ The resolved config file path, or ``None`` when neither source
66
+ supplies a non-empty path.
67
+ """
68
+ if cli_config is not None and cli_config.strip():
69
+ return cli_config
70
+
71
+ if env is None:
72
+ env = os.environ
73
+
74
+ env_value = env.get(ENV_CONFIG_VAR)
75
+ if env_value is not None and env_value.strip():
76
+ return env_value
77
+
78
+ return None
79
+
80
+
81
+ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
82
+ """Parse command-line arguments.
83
+
84
+ Args:
85
+ argv: Argument list excluding the program name. Defaults to
86
+ ``sys.argv[1:]`` when ``None``.
87
+
88
+ Returns:
89
+ A namespace with a ``config`` attribute (``str | None``) holding the
90
+ ``--config`` value, or ``None`` when the flag was omitted.
91
+ """
92
+ parser = argparse.ArgumentParser(
93
+ prog="wildweb",
94
+ description=(
95
+ "wildweb MCP server: exposes web scraping as an MCP tool over "
96
+ "stdio."
97
+ ),
98
+ )
99
+ parser.add_argument(
100
+ "--config",
101
+ dest="config",
102
+ default=None,
103
+ metavar="PATH",
104
+ help=(
105
+ "path to the JSON configuration file; overrides the "
106
+ f"{ENV_CONFIG_VAR} environment variable"
107
+ ),
108
+ )
109
+ return parser.parse_args(argv)
110
+
111
+
112
+ def run(config: Config) -> int:
113
+ """Hand off to the composition root and run the async MCP server.
114
+
115
+ This is the seam between the CLI and the server. The server module is
116
+ imported lazily so that importing :mod:`wildweb.cli` (and calling
117
+ :func:`resolve_config_path` / :func:`parse_args`) does not require the
118
+ server/composition root to exist yet. The composition root
119
+ (``wildweb.server.run_server``) builds the registry, registers the
120
+ Playwright scraper, starts it, serves over stdio, and releases resources on
121
+ shutdown.
122
+
123
+ Args:
124
+ config: The validated configuration driving the server.
125
+
126
+ Returns:
127
+ The process exit code (``0`` on a clean server shutdown).
128
+ """
129
+ import asyncio
130
+
131
+ from wildweb.server import run_server
132
+
133
+ return asyncio.run(run_server(config))
134
+
135
+
136
+ def main(argv: list[str] | None = None) -> int:
137
+ """CLI entry point.
138
+
139
+ Steps:
140
+
141
+ 1. Parse arguments (``argv`` defaults to ``sys.argv[1:]``).
142
+ 2. Resolve the config path by precedence (Requirements 6.4-6.6). When
143
+ neither the ``--config`` argument nor the ``WILDWEB_CONFIG`` environment
144
+ variable is supplied, fall back to the built-in :class:`Config`
145
+ defaults rather than failing.
146
+ 3. Otherwise, load and validate the config file (Requirements 5.3-5.5,
147
+ 7.5). On :class:`ConfigError`, write the message to stderr and return
148
+ non-zero.
149
+ 4. Configure logging from the resolved log level (Requirements 7.3, 7.4).
150
+ 5. Hand off to the composition root via :func:`run`.
151
+
152
+ Args:
153
+ argv: Argument list excluding the program name. Defaults to
154
+ ``sys.argv[1:]`` when ``None``.
155
+
156
+ Returns:
157
+ The process exit code. Used directly as the exit status by the
158
+ ``console_scripts`` entry point.
159
+ """
160
+ args = parse_args(argv)
161
+ config_path = resolve_config_path(args.config, os.environ)
162
+
163
+ if config_path is None:
164
+ config = Config()
165
+ else:
166
+ try:
167
+ config = ConfigLoader().load(config_path)
168
+ except ConfigError as exc:
169
+ print(f"error: {exc}", file=sys.stderr)
170
+ return EXIT_CONFIG_ERROR
171
+
172
+ configure_logging(config)
173
+
174
+ return run(config)
175
+
176
+
177
+ if __name__ == "__main__":
178
+ sys.exit(main())