streamwright-mcp 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- streamwright_mcp-0.0.4/PKG-INFO +56 -0
- streamwright_mcp-0.0.4/README.md +37 -0
- {streamwright_mcp-0.0.2 → streamwright_mcp-0.0.4}/pyproject.toml +11 -9
- streamwright_mcp-0.0.4/src/streamwright/mcp/__init__.py +3 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/cli.py +64 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/cli_reference.py +56 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/connectors.py +130 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/docs.py +69 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/engines.py +195 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/http_transport.py +73 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/inspection.py +84 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/report.py +55 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/schemas.py +62 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/server.py +186 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/validation.py +54 -0
- streamwright_mcp-0.0.4/src/streamwright/mcp/workspace.py +52 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp/__init__.py +13 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp/cli.py +5 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp/server.py +5 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp.egg-info/PKG-INFO +56 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp.egg-info/SOURCES.txt +35 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp.egg-info/entry_points.txt +5 -0
- streamwright_mcp-0.0.4/src/streamwright_mcp.egg-info/requires.txt +5 -0
- {streamwright_mcp-0.0.2 → streamwright_mcp-0.0.4}/src/streamwright_mcp.egg-info/top_level.txt +1 -0
- streamwright_mcp-0.0.4/tests/test_cli.py +83 -0
- streamwright_mcp-0.0.4/tests/test_cli_reference.py +29 -0
- streamwright_mcp-0.0.4/tests/test_connectors.py +126 -0
- streamwright_mcp-0.0.4/tests/test_engines.py +119 -0
- streamwright_mcp-0.0.4/tests/test_inspection.py +75 -0
- streamwright_mcp-0.0.4/tests/test_packaging.py +76 -0
- streamwright_mcp-0.0.4/tests/test_parity.py +138 -0
- streamwright_mcp-0.0.4/tests/test_schemas.py +71 -0
- streamwright_mcp-0.0.4/tests/test_validation.py +189 -0
- streamwright_mcp-0.0.4/tests/test_workspace.py +115 -0
- streamwright_mcp-0.0.2/PKG-INFO +0 -154
- streamwright_mcp-0.0.2/README.md +0 -131
- streamwright_mcp-0.0.2/src/streamwright_mcp/__init__.py +0 -5
- streamwright_mcp-0.0.2/src/streamwright_mcp/cli.py +0 -26
- streamwright_mcp-0.0.2/src/streamwright_mcp/contracts.py +0 -260
- streamwright_mcp-0.0.2/src/streamwright_mcp/hub.py +0 -84
- streamwright_mcp-0.0.2/src/streamwright_mcp/inspector.py +0 -300
- streamwright_mcp-0.0.2/src/streamwright_mcp/server.py +0 -154
- streamwright_mcp-0.0.2/src/streamwright_mcp/validator.py +0 -99
- streamwright_mcp-0.0.2/src/streamwright_mcp.egg-info/PKG-INFO +0 -154
- streamwright_mcp-0.0.2/src/streamwright_mcp.egg-info/SOURCES.txt +0 -20
- streamwright_mcp-0.0.2/src/streamwright_mcp.egg-info/entry_points.txt +0 -2
- streamwright_mcp-0.0.2/src/streamwright_mcp.egg-info/requires.txt +0 -9
- streamwright_mcp-0.0.2/tests/test_contracts.py +0 -127
- streamwright_mcp-0.0.2/tests/test_inspector.py +0 -67
- streamwright_mcp-0.0.2/tests/test_server.py +0 -88
- streamwright_mcp-0.0.2/tests/test_validator.py +0 -19
- {streamwright_mcp-0.0.2 → streamwright_mcp-0.0.4}/LICENSE +0 -0
- {streamwright_mcp-0.0.2 → streamwright_mcp-0.0.4}/setup.cfg +0 -0
- {streamwright_mcp-0.0.2 → streamwright_mcp-0.0.4}/src/streamwright_mcp.egg-info/dependency_links.txt +0 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: streamwright-mcp
|
|
3
|
+
Version: 0.0.4
|
|
4
|
+
Summary: Model Context Protocol (MCP) server for StreamWright declarative ELT framework
|
|
5
|
+
Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://streamwright.web.app
|
|
8
|
+
Project-URL: Documentation, https://streamwright.web.app/docs/ai/mcp
|
|
9
|
+
Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright-mcp
|
|
10
|
+
Project-URL: Issue Tracker, https://github.com/karthick-jaganathan/streamwright-mcp/issues
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: mcp<3,>=2.3
|
|
15
|
+
Requires-Dist: streamwright~=0.1.11
|
|
16
|
+
Provides-Extra: test
|
|
17
|
+
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# StreamWright MCP
|
|
21
|
+
|
|
22
|
+
[](https://pypi.org/project/streamwright-mcp/)
|
|
23
|
+
[](https://pypi.org/project/streamwright-mcp/)
|
|
24
|
+
[](https://github.com/karthick-jaganathan/streamwright-mcp/blob/main/LICENSE)
|
|
25
|
+
[](https://streamwright.web.app/docs/ai/mcp/)
|
|
26
|
+
|
|
27
|
+
A [Model Context Protocol](https://modelcontextprotocol.io) server for [StreamWright](https://pypi.org/project/streamwright/).
|
|
28
|
+
AI assistants use it to discover connectors, read their contracts and the engine guides, and validate sources and
|
|
29
|
+
jobs. It never runs a pipeline and never reads secret values.
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install streamwright-mcp
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{
|
|
39
|
+
"mcpServers": {
|
|
40
|
+
"streamwright": {"command": "streamwright-mcp"}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
stdio by default; `--transport sse | streamable-http` needs `--allow-http` and a token in `STREAMWRIGHT_MCP_TOKEN`.
|
|
46
|
+
|
|
47
|
+
## Tools
|
|
48
|
+
|
|
49
|
+
`list_connectors` · `get_connector_contract` · `get_connector_guide` · `get_core_schema` · `get_engine_guide` ·
|
|
50
|
+
`get_cli_reference` · `validate_source_config` · `inspect_stream_transform` · `validate_duckdb_job` · `validate_spark_job`
|
|
51
|
+
|
|
52
|
+
## Links
|
|
53
|
+
|
|
54
|
+
[Documentation](https://streamwright.web.app/docs/ai/mcp/) ·
|
|
55
|
+
[Tool reference](https://github.com/karthick-jaganathan/streamwright-mcp/blob/main/docs/mcp.md) ·
|
|
56
|
+
[Source code](https://github.com/karthick-jaganathan/streamwright-mcp)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# StreamWright MCP
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/streamwright-mcp/)
|
|
4
|
+
[](https://pypi.org/project/streamwright-mcp/)
|
|
5
|
+
[](https://github.com/karthick-jaganathan/streamwright-mcp/blob/main/LICENSE)
|
|
6
|
+
[](https://streamwright.web.app/docs/ai/mcp/)
|
|
7
|
+
|
|
8
|
+
A [Model Context Protocol](https://modelcontextprotocol.io) server for [StreamWright](https://pypi.org/project/streamwright/).
|
|
9
|
+
AI assistants use it to discover connectors, read their contracts and the engine guides, and validate sources and
|
|
10
|
+
jobs. It never runs a pipeline and never reads secret values.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install streamwright-mcp
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{
|
|
20
|
+
"mcpServers": {
|
|
21
|
+
"streamwright": {"command": "streamwright-mcp"}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
stdio by default; `--transport sse | streamable-http` needs `--allow-http` and a token in `STREAMWRIGHT_MCP_TOKEN`.
|
|
27
|
+
|
|
28
|
+
## Tools
|
|
29
|
+
|
|
30
|
+
`list_connectors` · `get_connector_contract` · `get_connector_guide` · `get_core_schema` · `get_engine_guide` ·
|
|
31
|
+
`get_cli_reference` · `validate_source_config` · `inspect_stream_transform` · `validate_duckdb_job` · `validate_spark_job`
|
|
32
|
+
|
|
33
|
+
## Links
|
|
34
|
+
|
|
35
|
+
[Documentation](https://streamwright.web.app/docs/ai/mcp/) ·
|
|
36
|
+
[Tool reference](https://github.com/karthick-jaganathan/streamwright-mcp/blob/main/docs/mcp.md) ·
|
|
37
|
+
[Source code](https://github.com/karthick-jaganathan/streamwright-mcp)
|
|
@@ -4,21 +4,17 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "streamwright-mcp"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.4"
|
|
8
8
|
description = "Model Context Protocol (MCP) server for StreamWright declarative ELT framework"
|
|
9
9
|
authors = [
|
|
10
10
|
{name = "Karthick Jaganathan", email = "kjaganathan.sde@gmail.com"},
|
|
11
11
|
]
|
|
12
12
|
readme = "README.md"
|
|
13
|
-
requires-python = ">=3.
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
14
|
license = { text = "Apache-2.0" }
|
|
15
15
|
dependencies = [
|
|
16
|
-
"mcp>=
|
|
17
|
-
"streamwright
|
|
18
|
-
"pyyaml>=6.0.0",
|
|
19
|
-
"pydantic>=2.0.0",
|
|
20
|
-
"requests>=2.25.0",
|
|
21
|
-
"sqlglot>=20.0.0",
|
|
16
|
+
"mcp>=2.3,<3",
|
|
17
|
+
"streamwright~=0.1.11",
|
|
22
18
|
]
|
|
23
19
|
|
|
24
20
|
[project.urls]
|
|
@@ -28,7 +24,10 @@ Repository = "https://github.com/karthick-jaganathan/streamwright-mcp"
|
|
|
28
24
|
"Issue Tracker" = "https://github.com/karthick-jaganathan/streamwright-mcp/issues"
|
|
29
25
|
|
|
30
26
|
[project.scripts]
|
|
31
|
-
streamwright-mcp = "
|
|
27
|
+
streamwright-mcp = "streamwright.mcp.cli:main"
|
|
28
|
+
|
|
29
|
+
[project.entry-points."streamwright.commands"]
|
|
30
|
+
mcp = "streamwright.mcp.cli:register"
|
|
32
31
|
|
|
33
32
|
[project.optional-dependencies]
|
|
34
33
|
test = [
|
|
@@ -36,4 +35,7 @@ test = [
|
|
|
36
35
|
]
|
|
37
36
|
|
|
38
37
|
[tool.setuptools.packages.find]
|
|
38
|
+
# `streamwright` is a namespace package shared with streamwright (core), streamwright-spark and the connectors;
|
|
39
|
+
# `streamwright_mcp` is the deprecated alias of streamwright.mcp
|
|
39
40
|
where = ["src"]
|
|
41
|
+
include = ["streamwright.mcp*", "streamwright_mcp*"]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command line of the StreamWright MCP server: the `streamwright-mcp` script and the `streamwright mcp` subcommand
|
|
3
|
+
(registered through the `streamwright.commands` entry-point group).
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import os
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
TOKEN_ENV = "STREAMWRIGHT_MCP_TOKEN"
|
|
11
|
+
TRANSPORTS = ("stdio", "sse", "streamable-http")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def add_arguments(parser: argparse.ArgumentParser) -> None:
|
|
15
|
+
parser.add_argument("--transport", choices=TRANSPORTS, default="stdio",
|
|
16
|
+
help="MCP transport protocol (default: stdio)")
|
|
17
|
+
parser.add_argument("--allow-http", action="store_true",
|
|
18
|
+
help="required for the sse and streamable-http transports, which serve over the network; "
|
|
19
|
+
f"callers must send the bearer token in {TOKEN_ENV}")
|
|
20
|
+
parser.add_argument("--host", default="127.0.0.1",
|
|
21
|
+
help="interface the HTTP transports bind to (default: 127.0.0.1)")
|
|
22
|
+
parser.add_argument("--port", type=int, default=8000, help="port the HTTP transports listen on (default: 8000)")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
26
|
+
parser = argparse.ArgumentParser(prog="streamwright-mcp",
|
|
27
|
+
description="StreamWright Model Context Protocol (MCP) server.")
|
|
28
|
+
add_arguments(parser)
|
|
29
|
+
return parser
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def serve(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
33
|
+
from streamwright.mcp.server import server # noqa: PLC0415 loads every job engine: only when serving
|
|
34
|
+
|
|
35
|
+
if args.transport == "stdio":
|
|
36
|
+
server.run(transport="stdio")
|
|
37
|
+
return 0
|
|
38
|
+
if not args.allow_http:
|
|
39
|
+
parser.error(f"--transport {args.transport} serves over the network; pass --allow-http to enable it")
|
|
40
|
+
token = os.environ.get(TOKEN_ENV, "")
|
|
41
|
+
if not token:
|
|
42
|
+
parser.error(f"--transport {args.transport} requires a bearer token in the {TOKEN_ENV} environment variable")
|
|
43
|
+
|
|
44
|
+
from streamwright.mcp.http_transport import create_http_app, serve_http_app # noqa: PLC0415 imports the server
|
|
45
|
+
|
|
46
|
+
serve_http_app(create_http_app(args.transport, args.host, args.port, token), args.host, args.port)
|
|
47
|
+
return 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def register(subparsers) -> None:
|
|
51
|
+
"""`streamwright.commands` entry point: adds `streamwright mcp`."""
|
|
52
|
+
parser = subparsers.add_parser("mcp", help="run the StreamWright Model Context Protocol (MCP) server",
|
|
53
|
+
description="Run the StreamWright Model Context Protocol (MCP) server.")
|
|
54
|
+
add_arguments(parser)
|
|
55
|
+
parser.set_defaults(command_handler=lambda args: serve(args, parser))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
59
|
+
parser = build_parser()
|
|
60
|
+
return serve(parser.parse_args(argv), parser)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
if __name__ == "__main__":
|
|
64
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""`get_cli_reference`: the live help of the `streamwright` commands, read from the CLI's own parsers."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from mcp.server.mcpserver.exceptions import ToolError
|
|
8
|
+
from streamwright.core.cli import command_parsers
|
|
9
|
+
|
|
10
|
+
SHARED_JOB_OPTIONS = "job" # command_parsers() publishes the options every job command takes under this key
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _children(parser: argparse.ArgumentParser) -> dict[str, argparse.ArgumentParser]:
|
|
14
|
+
for action in parser._actions:
|
|
15
|
+
if isinstance(action, argparse._SubParsersAction):
|
|
16
|
+
return dict(action.choices)
|
|
17
|
+
return {}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _usage_line(parser: argparse.ArgumentParser) -> str:
|
|
21
|
+
return parser.format_usage().strip()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _describe(words: list[str], parser: argparse.ArgumentParser) -> dict[str, Any]:
|
|
25
|
+
children = _children(parser)
|
|
26
|
+
return {
|
|
27
|
+
"command": " ".join(words),
|
|
28
|
+
"usage": _usage_line(parser),
|
|
29
|
+
"help_text": parser.format_help(),
|
|
30
|
+
"subcommands": {name: _usage_line(child) for name, child in children.items()} or None,
|
|
31
|
+
"run": f"streamwright {' '.join(words)} --help",
|
|
32
|
+
"fallback": f"{sys.executable} -m streamwright.core.cli {' '.join(words)} --help",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_cli_reference(command: str | None = None) -> dict[str, Any]:
|
|
37
|
+
"""The commands (no `command`), or the usage and help of one command such as `validate` or `duckdb run`."""
|
|
38
|
+
parsers = command_parsers()
|
|
39
|
+
job_options = parsers.pop(SHARED_JOB_OPTIONS, None)
|
|
40
|
+
words = (command or "").split()
|
|
41
|
+
if not words:
|
|
42
|
+
return {
|
|
43
|
+
"binary": "streamwright",
|
|
44
|
+
"alias": "sw",
|
|
45
|
+
"commands": {name: _usage_line(parser) for name, parser in sorted(parsers.items())},
|
|
46
|
+
"job_options": job_options.format_help() if job_options else None,
|
|
47
|
+
"run": "streamwright --help",
|
|
48
|
+
"fallback": f"{sys.executable} -m streamwright.core.cli --help",
|
|
49
|
+
}
|
|
50
|
+
parser, path = parsers.get(words[0]), words[:1]
|
|
51
|
+
for word in words[1:]:
|
|
52
|
+
parser = _children(parser).get(word) if parser is not None else None
|
|
53
|
+
path.append(word)
|
|
54
|
+
if parser is None:
|
|
55
|
+
raise ToolError(f"Unknown command '{command}'; commands: {', '.join(sorted(parsers))}.")
|
|
56
|
+
return _describe(path, parser)
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Connector listing and contracts: installed connectors from their definitions, contracts from the engines."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from streamwright.core import catalog
|
|
7
|
+
from streamwright.core.runtime import components
|
|
8
|
+
|
|
9
|
+
from streamwright.mcp.engines import SOURCE_KIND, EngineDescriptor, EngineError, EngineRegistry, paged_guide
|
|
10
|
+
|
|
11
|
+
_CONNECTOR_NAME = re.compile(r"[a-z][a-z0-9_]{0,62}")
|
|
12
|
+
_INSTALL_COMMAND = "streamwright connectors install {name}"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def is_valid_connector_name(name: Any) -> bool:
|
|
16
|
+
"""Whether `name` is a plain connector identifier."""
|
|
17
|
+
return isinstance(name, str) and _CONNECTOR_NAME.fullmatch(name) is not None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _install_details(name: str, entry: dict[str, Any] | None) -> dict[str, Any]:
|
|
21
|
+
details: dict[str, Any] = {"install_command": _INSTALL_COMMAND.format(name=name)}
|
|
22
|
+
if entry and entry.get("install"):
|
|
23
|
+
details["install"] = entry["install"]
|
|
24
|
+
return details
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _catalog_entries() -> dict[str, dict[str, Any]]:
|
|
28
|
+
return dict(catalog.connectors())
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _select_engines(registry: EngineRegistry, engine: str | None) -> list[EngineDescriptor]:
|
|
32
|
+
if not engine:
|
|
33
|
+
return list(registry)
|
|
34
|
+
found = registry.find(engine)
|
|
35
|
+
if found is None:
|
|
36
|
+
raise EngineError(f"Unknown engine '{engine}'; installed engines: {', '.join(registry.kinds())}.")
|
|
37
|
+
return [found]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def list_connectors(registry: EngineRegistry, include_hub: bool = False,
|
|
41
|
+
engine: str | None = None) -> list[dict[str, Any]]:
|
|
42
|
+
"""
|
|
43
|
+
Installed connectors with the engines each one supports (from the connector definitions: only their specs are
|
|
44
|
+
imported); with `include_hub`, also the catalog connectors that are not installed (with how to install them).
|
|
45
|
+
"""
|
|
46
|
+
engines = _select_engines(registry, engine)
|
|
47
|
+
found: dict[str, dict[str, Any]] = {}
|
|
48
|
+
for descriptor in engines:
|
|
49
|
+
for item in descriptor.connectors():
|
|
50
|
+
entry = found.setdefault(item["name"], {"name": item["name"], "installed": True,
|
|
51
|
+
"distribution": item["distribution"], "version": item["version"],
|
|
52
|
+
"engines": [], "reach": {}})
|
|
53
|
+
entry["engines"].append(descriptor.kind)
|
|
54
|
+
entry["reach"][descriptor.kind] = item.get("support", "native") # native adapter or a call step
|
|
55
|
+
if include_hub:
|
|
56
|
+
entries = _catalog_entries()
|
|
57
|
+
for name, entry in entries.items():
|
|
58
|
+
if name in found:
|
|
59
|
+
found[name].update({"title": entry.get("title"), "family": entry.get("family"),
|
|
60
|
+
"summary": entry.get("summary")})
|
|
61
|
+
elif not engine:
|
|
62
|
+
found[name] = {"name": name, "installed": False, "title": entry.get("title"),
|
|
63
|
+
"family": entry.get("family"), "summary": entry.get("summary"),
|
|
64
|
+
**_install_details(name, entry)}
|
|
65
|
+
return [found[name] for name in sorted(found)]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def get_connector_contract(registry: EngineRegistry, name: str, engine: str | None = None) -> dict[str, Any]:
|
|
69
|
+
"""
|
|
70
|
+
The contract of connector `name` for each engine that has an adapter for it, as the engines declare it
|
|
71
|
+
(nothing is connected). When it is not installed: how to install it.
|
|
72
|
+
"""
|
|
73
|
+
if not is_valid_connector_name(name):
|
|
74
|
+
raise EngineError("Invalid connector name: expected lowercase letters, digits and '_', starting with a "
|
|
75
|
+
"letter (max 63 characters).")
|
|
76
|
+
engines = _select_engines(registry, engine)
|
|
77
|
+
contracts: dict[str, Any] = {}
|
|
78
|
+
for descriptor in engines:
|
|
79
|
+
reach = {item["name"]: item.get("support", "native") for item in descriptor.connectors()}
|
|
80
|
+
if name not in reach:
|
|
81
|
+
continue
|
|
82
|
+
if reach[name] == "call": # reachable through a source-adapter call step
|
|
83
|
+
source = registry.find(SOURCE_KIND)
|
|
84
|
+
described = source.describe_connector(name) if source is not None else None
|
|
85
|
+
contracts[descriptor.kind] = {"via": "call step", **(described or {})}
|
|
86
|
+
continue
|
|
87
|
+
description = descriptor.describe_connector(name)
|
|
88
|
+
contracts[descriptor.kind] = description or {
|
|
89
|
+
"error": f"Connector '{name}' is installed but could not be loaded."}
|
|
90
|
+
if contracts:
|
|
91
|
+
guide = _guide_text(name)
|
|
92
|
+
sections = {"guide_sections": paged_guide(guide)(None)["sections"]} if guide else {}
|
|
93
|
+
return {"name": name, "installed": True, "engines": contracts, **sections}
|
|
94
|
+
installed_elsewhere = [d.kind for d in registry if name in {item["name"] for item in d.connectors()}]
|
|
95
|
+
if installed_elsewhere:
|
|
96
|
+
return {"name": name, "installed": True, "engines": {},
|
|
97
|
+
"message": f"Connector '{name}' ships no adapter for {engine}; it supports: "
|
|
98
|
+
f"{', '.join(installed_elsewhere)}."}
|
|
99
|
+
result = {"name": name, "installed": False,
|
|
100
|
+
"message": f"Connector '{name}' is not installed in the current Python environment."}
|
|
101
|
+
try:
|
|
102
|
+
entry = _catalog_entries().get(name)
|
|
103
|
+
except RuntimeError as exc:
|
|
104
|
+
entry = None
|
|
105
|
+
result["catalog_error"] = str(exc)
|
|
106
|
+
return {**result, **_install_details(name, entry)}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _guide_text(name: str) -> str | None:
|
|
110
|
+
try:
|
|
111
|
+
return components.guide(name)
|
|
112
|
+
except components.ComponentLoadError:
|
|
113
|
+
return None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def get_connector_guide(name: str, section: str | None = None) -> dict[str, Any]:
|
|
117
|
+
"""
|
|
118
|
+
The user guide an installed connector ships (examples, query builders, vendor arguments, engine notes, with the
|
|
119
|
+
tables generated from its definition), paged: without `section` the section titles, otherwise that section.
|
|
120
|
+
"""
|
|
121
|
+
if not is_valid_connector_name(name):
|
|
122
|
+
raise EngineError("Invalid connector name: expected lowercase letters, digits and '_', starting with a "
|
|
123
|
+
"letter (max 63 characters).")
|
|
124
|
+
if name not in components.available():
|
|
125
|
+
raise EngineError(f"Connector '{name}' is not installed; get_connector_contract('{name}') says how to "
|
|
126
|
+
"install it.")
|
|
127
|
+
text = _guide_text(name)
|
|
128
|
+
if not text:
|
|
129
|
+
raise EngineError(f"Connector '{name}' ships no guide; get_connector_contract('{name}') has its contract.")
|
|
130
|
+
return {"name": name, **paged_guide(text)(section)}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Generates the tool and resource reference of docs/mcp.md from the server's registry:
|
|
3
|
+
|
|
4
|
+
python -m streamwright.mcp.docs [--check] DOCUMENT...
|
|
5
|
+
|
|
6
|
+
Each DOCUMENT holds a block between `<!-- mcp-reference:start -->` and `<!-- mcp-reference:end -->`. The reference is
|
|
7
|
+
rendered for the `source` engine; job engines add one `validate_<engine>_job` tool and one set of schema and guide
|
|
8
|
+
resources each, rendered here once from the same templates.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import asyncio
|
|
13
|
+
import inspect
|
|
14
|
+
import sys
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from streamwright.mcp.engines import EngineRegistry, source_engine
|
|
18
|
+
from streamwright.mcp.server import JOB_TOOL_DESCRIPTION, build_server
|
|
19
|
+
|
|
20
|
+
START = "<!-- mcp-reference:start -->"
|
|
21
|
+
END = "<!-- mcp-reference:end -->"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def render_reference() -> str:
|
|
25
|
+
server = build_server(EngineRegistry([source_engine()]))
|
|
26
|
+
lines = ["#### Tools", ""]
|
|
27
|
+
for tool in asyncio.run(server.list_tools()):
|
|
28
|
+
lines += [f"##### `{tool.name}`", "", inspect.cleandoc(tool.description or ""), ""]
|
|
29
|
+
lines += ["##### `validate_<engine>_job`", "",
|
|
30
|
+
"One tool per installed job engine (for example `validate_duckdb_job`, `validate_spark_job`), generated "
|
|
31
|
+
"from the engine registry.", "",
|
|
32
|
+
inspect.cleandoc(JOB_TOOL_DESCRIPTION.format(kind="<Kind>", title="<engine title>")), "",
|
|
33
|
+
"#### Resources", ""]
|
|
34
|
+
for resource in asyncio.run(server.list_resources()):
|
|
35
|
+
lines.append(f"- `{resource.uri}`: {resource.description}")
|
|
36
|
+
for template in asyncio.run(server.list_resource_templates()):
|
|
37
|
+
lines.append(f"- `{template.uri_template}`: {template.description}")
|
|
38
|
+
lines += ["- `streamwright://schema/<name>`, `streamwright://guide/<engine>` and "
|
|
39
|
+
"`streamwright://guide/<engine>/{section}`: the same for every installed job engine."]
|
|
40
|
+
return "\n".join(lines) + "\n"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def replace_block(text: str, block: str) -> str:
|
|
44
|
+
start, end = text.index(START) + len(START), text.index(END)
|
|
45
|
+
return f"{text[:start]}\n\n{block}\n{text[end:]}"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def main(argv: list[str] | None = None) -> int:
|
|
49
|
+
parser = argparse.ArgumentParser(description="Generate the MCP tool reference in documentation documents.")
|
|
50
|
+
parser.add_argument("--check", action="store_true",
|
|
51
|
+
help="fail if a document is out of date instead of rewriting it")
|
|
52
|
+
parser.add_argument("documents", nargs="+", type=Path)
|
|
53
|
+
args = parser.parse_args(argv)
|
|
54
|
+
block, stale = render_reference(), []
|
|
55
|
+
for path in args.documents:
|
|
56
|
+
current = path.read_text(encoding="utf-8")
|
|
57
|
+
updated = replace_block(current, block)
|
|
58
|
+
if updated != current:
|
|
59
|
+
stale.append(str(path))
|
|
60
|
+
if not args.check:
|
|
61
|
+
path.write_text(updated, encoding="utf-8")
|
|
62
|
+
if stale and args.check:
|
|
63
|
+
sys.stderr.write("out of date (run python -m streamwright.mcp.docs): " + ", ".join(stale) + "\n")
|
|
64
|
+
return 1
|
|
65
|
+
return 0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
if __name__ == "__main__":
|
|
69
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The engines the MCP server can describe and validate, discovered from installed plug-ins.
|
|
3
|
+
|
|
4
|
+
An engine is `source` (core's own) or a job kind registered in the `streamwright.job_kinds` entry-point group, whose
|
|
5
|
+
entry point is a `streamwright.core.jobs.engine.JobEngine` object: the server uses only that protocol.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import inspect
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from mcp.server.mcpserver.exceptions import ResourceError, ToolError
|
|
16
|
+
from streamwright.core.jobs.engine import JobEngine, engine_name, load_job_engine
|
|
17
|
+
from streamwright.core.runtime import components, plugins
|
|
18
|
+
from streamwright.core.runtime.definitions import engine_connectors
|
|
19
|
+
from streamwright.core.validation import Issue
|
|
20
|
+
from streamwright.core.validation import schema as source_schema
|
|
21
|
+
from streamwright.core.validation.engine import check_components, is_valid, validate_source, validate_text
|
|
22
|
+
|
|
23
|
+
from streamwright.mcp.report import Report
|
|
24
|
+
|
|
25
|
+
SOURCE_KIND = "source"
|
|
26
|
+
_HEADING = re.compile(r"^#{2,3} +(.+?) *$", re.MULTILINE)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class EngineError(ToolError, ResourceError):
|
|
30
|
+
"""A request names an engine, schema or guide section that is not available (the message reaches the client)."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class EngineDescriptor:
|
|
35
|
+
"""What every tool and resource needs from one engine; the only per-engine code path of the server."""
|
|
36
|
+
|
|
37
|
+
kind: str
|
|
38
|
+
key: str
|
|
39
|
+
title: str
|
|
40
|
+
schemas: Mapping[str, Callable[[], dict[str, Any]]]
|
|
41
|
+
guide: Callable[[str | None], dict[str, Any]]
|
|
42
|
+
validate: Callable[..., Report]
|
|
43
|
+
describe_connector: Callable[[str], dict[str, Any] | None]
|
|
44
|
+
|
|
45
|
+
def connectors(self) -> list[dict[str, Any]]:
|
|
46
|
+
"""Name, distribution and version of the installed connectors with an adapter for this engine (their
|
|
47
|
+
definitions say so; only the connectors' specs are imported)."""
|
|
48
|
+
return engine_connectors(self.key)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class EngineRegistry:
|
|
52
|
+
"""The descriptors of the installed engines, by kind, and the engines that could not be loaded."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, engines: list[EngineDescriptor], problems: dict[str, str] | None = None):
|
|
55
|
+
self._engines = {engine.kind: engine for engine in engines}
|
|
56
|
+
self.problems = dict(problems or {})
|
|
57
|
+
|
|
58
|
+
def __iter__(self) -> Iterator[EngineDescriptor]:
|
|
59
|
+
return iter(self._engines.values())
|
|
60
|
+
|
|
61
|
+
def kinds(self) -> list[str]:
|
|
62
|
+
return list(self._engines)
|
|
63
|
+
|
|
64
|
+
def job_engines(self) -> list[EngineDescriptor]:
|
|
65
|
+
return [engine for engine in self if engine.kind != SOURCE_KIND]
|
|
66
|
+
|
|
67
|
+
def get(self, kind: str) -> EngineDescriptor | None:
|
|
68
|
+
return self._engines.get(kind)
|
|
69
|
+
|
|
70
|
+
def find(self, name: str) -> EngineDescriptor | None:
|
|
71
|
+
"""The engine named by its kind (`DuckDBJob`) or key (`duckdb`), ignoring case."""
|
|
72
|
+
wanted = (name or "").strip().lower()
|
|
73
|
+
for engine in self:
|
|
74
|
+
if wanted in (engine.kind.lower(), engine.key):
|
|
75
|
+
return engine
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
def find_schema(self, name: str) -> tuple[EngineDescriptor, str] | None:
|
|
79
|
+
"""The engine and schema name for `name`: an engine, or a schema such as `stream`."""
|
|
80
|
+
wanted = (name or "").strip().lower()
|
|
81
|
+
for engine in self:
|
|
82
|
+
if wanted in (engine.kind.lower(), engine.key):
|
|
83
|
+
return engine, next(iter(engine.schemas))
|
|
84
|
+
if wanted in engine.schemas:
|
|
85
|
+
return engine, wanted
|
|
86
|
+
return None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def split_sections(text: str) -> dict[str, str]:
|
|
90
|
+
"""A markdown document by heading: {heading: text of the section}."""
|
|
91
|
+
matches = list(_HEADING.finditer(text))
|
|
92
|
+
sections = {}
|
|
93
|
+
for position, match in enumerate(matches):
|
|
94
|
+
end = matches[position + 1].start() if position + 1 < len(matches) else len(text)
|
|
95
|
+
sections[match.group(1)] = text[match.start():end].strip()
|
|
96
|
+
return sections
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def paged_guide(text: str) -> Callable[[str | None], dict[str, Any]]:
|
|
100
|
+
"""A guide(section) over a markdown document: the section titles, or one section found by title."""
|
|
101
|
+
sections = split_sections(text)
|
|
102
|
+
|
|
103
|
+
def guide(section: str | None = None) -> dict[str, Any]:
|
|
104
|
+
if not section:
|
|
105
|
+
return {"sections": list(sections)}
|
|
106
|
+
wanted = section.strip().lower()
|
|
107
|
+
for title, body in sections.items():
|
|
108
|
+
if wanted in title.lower():
|
|
109
|
+
return {"section": title, "text": body}
|
|
110
|
+
raise EngineError(f"unknown guide section '{section}'; sections: {', '.join(sections)}")
|
|
111
|
+
|
|
112
|
+
return guide
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _source_report(issues: list[Issue], strict: bool, path: str | None) -> Report:
|
|
116
|
+
subject = {"name": os.path.basename(path.rstrip(os.sep)), "path": path} if path else None
|
|
117
|
+
return Report(SOURCE_KIND, is_valid(issues, strict=strict), issues, subject)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _validate_source(path=None, content=None, overlays=(), config=None, strict=False) -> Report:
|
|
121
|
+
if overlays or config:
|
|
122
|
+
raise ValueError("a source takes neither overlays nor config values")
|
|
123
|
+
if content is not None:
|
|
124
|
+
return _source_report(validate_text(content, source_check=check_components), strict, None)
|
|
125
|
+
return _source_report(validate_source(path, source_check=check_components), strict, path)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def source_engine() -> EngineDescriptor:
|
|
129
|
+
return EngineDescriptor(
|
|
130
|
+
kind=SOURCE_KIND,
|
|
131
|
+
key=SOURCE_KIND,
|
|
132
|
+
title="Source (kind: source): source.yaml and streams/*.yaml, run by the source engine",
|
|
133
|
+
schemas={"source": source_schema.json_schema, "stream": source_schema.stream_json_schema},
|
|
134
|
+
guide=paged_guide(inspect.getdoc(source_schema) or ""),
|
|
135
|
+
validate=_validate_source,
|
|
136
|
+
describe_connector=components.describe,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _job_subject(job: Mapping[str, Any] | None, path: str | None) -> dict[str, Any] | None:
|
|
141
|
+
if job is None:
|
|
142
|
+
return {"path": path} if path else None
|
|
143
|
+
steps, outputs, connections = job.get("steps"), job.get("outputs"), job.get("connections")
|
|
144
|
+
subject = {
|
|
145
|
+
"name": job.get("name"),
|
|
146
|
+
"kind": job.get("kind"),
|
|
147
|
+
"step_count": len(steps) if isinstance(steps, list) else 0,
|
|
148
|
+
"output_count": len(outputs) if isinstance(outputs, list) else 0,
|
|
149
|
+
"connection_count": len(connections) if isinstance(connections, dict) else 0,
|
|
150
|
+
}
|
|
151
|
+
if path:
|
|
152
|
+
subject["path"] = path
|
|
153
|
+
return subject
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _job_report(kind: str, result: Any, path: str | None) -> Report:
|
|
157
|
+
return Report(kind, result.valid, list(result.issues), _job_subject(result.job, path), result.plan)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def job_engine(engine: JobEngine) -> EngineDescriptor:
|
|
161
|
+
kind = engine.kind
|
|
162
|
+
key = engine_name(kind)
|
|
163
|
+
|
|
164
|
+
def validate(path=None, content=None, overlays=(), config=None, strict=False) -> Report:
|
|
165
|
+
if content is not None:
|
|
166
|
+
result = engine.validate_job_text(content, overlays=overlays, config=config, strict=strict)
|
|
167
|
+
else:
|
|
168
|
+
result = engine.validate_job(path, overlays=overlays, config=config, strict=strict)
|
|
169
|
+
return _job_report(kind, result, path)
|
|
170
|
+
|
|
171
|
+
def describe_connector(name: str) -> dict[str, Any] | None:
|
|
172
|
+
description = engine.describe_connector(name)
|
|
173
|
+
return None if description is None else {"name": name, "kind": "connector", "engine": kind, **description}
|
|
174
|
+
|
|
175
|
+
return EngineDescriptor(
|
|
176
|
+
kind=kind,
|
|
177
|
+
key=key,
|
|
178
|
+
title=f"{kind}: a job manifest run by the {key} engine",
|
|
179
|
+
schemas={key: engine.json_schema},
|
|
180
|
+
guide=paged_guide(engine.guide()),
|
|
181
|
+
validate=validate,
|
|
182
|
+
describe_connector=describe_connector,
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def discover_engines(entry_points=None) -> EngineRegistry:
|
|
187
|
+
"""The `source` engine and one engine per installed job kind (a kind that cannot be loaded is a problem)."""
|
|
188
|
+
engines, problems = [source_engine()], {}
|
|
189
|
+
for entry in plugins.discover(plugins.JOB_KINDS, entry_points=entry_points):
|
|
190
|
+
kind = entry["name"]
|
|
191
|
+
try:
|
|
192
|
+
engines.append(job_engine(load_job_engine(kind, entry_points=entry_points)))
|
|
193
|
+
except plugins.PluginError as exc:
|
|
194
|
+
problems[kind] = str(exc)
|
|
195
|
+
return EngineRegistry(engines, problems)
|