streamwright 0.1.6__tar.gz → 0.1.7__tar.gz

{streamwright-0.1.6/src/streamwright.egg-info → streamwright-0.1.7}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.6
+Version: 0.1.7
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
 License: Apache-2.0
-Project-URL: Homepage, https://github.com/karthick-jaganathan/streamwright
-Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright
+Project-URL: Homepage, https://streamwright.web.app
 Project-URL: Documentation, https://streamwright.web.app/docs/
+Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright
+Project-URL: Issue Tracker, https://github.com/karthick-jaganathan/streamwright/issues
 Keywords: data,pipeline,etl,connector,singer,yaml,duckdb,rest-api
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
@@ -44,9 +45,12 @@ Each stream follows a three-stage pipeline:
 
 ## CLI Commands
 
-- **`streamwright run`** — Execute a source and stream records to the configured destination.
-- **`streamwright validate`** — Validate source definitions, authentication specifications, and schemas without executing network requests.
-- **`streamwright connectors`** — Browse, inspect, and install connectors from the StreamWright Hub.
+All commands can be invoked interchangeably using `streamwright` or the shorthand `sw` alias (e.g. `sw run`, `sw validate`, `sw connectors`, `sw mcp`).
+
+- **`streamwright run`** (or **`sw run`**) — Execute a source and stream records to the configured destination.
+- **`streamwright validate`** (or **`sw validate`**) — Validate source definitions, authentication specifications, and schemas without executing network requests.
+- **`streamwright connectors`** (or **`sw connectors`**) — Browse, inspect, and install connectors from the StreamWright Hub.
+- **`streamwright mcp`** (or **`sw mcp`**) — Launch the Model Context Protocol (MCP) server for AI coding assistants.
 
 ## Source Structure
 
@@ -120,3 +124,4 @@ streamwright run streamwright/examples/sources/readers/files_demo \
 - **Full documentation:** https://streamwright.web.app/docs/
 - **Architecture Guide:** https://streamwright.web.app/docs/architecture/
 - **Connectors & Readers:** https://streamwright.web.app/docs/connectors/
+- **MCP Server Guide:** https://streamwright.web.app/docs/ai/mcp/

{streamwright-0.1.6 → streamwright-0.1.7}/README.md

@@ -16,9 +16,12 @@ Each stream follows a three-stage pipeline:
 
 ## CLI Commands
 
-- **`streamwright run`** — Execute a source and stream records to the configured destination.
-- **`streamwright validate`** — Validate source definitions, authentication specifications, and schemas without executing network requests.
-- **`streamwright connectors`** — Browse, inspect, and install connectors from the StreamWright Hub.
+All commands can be invoked interchangeably using `streamwright` or the shorthand `sw` alias (e.g. `sw run`, `sw validate`, `sw connectors`, `sw mcp`).
+
+- **`streamwright run`** (or **`sw run`**) — Execute a source and stream records to the configured destination.
+- **`streamwright validate`** (or **`sw validate`**) — Validate source definitions, authentication specifications, and schemas without executing network requests.
+- **`streamwright connectors`** (or **`sw connectors`**) — Browse, inspect, and install connectors from the StreamWright Hub.
+- **`streamwright mcp`** (or **`sw mcp`**) — Launch the Model Context Protocol (MCP) server for AI coding assistants.
 
 ## Source Structure
 
@@ -92,3 +95,4 @@ streamwright run streamwright/examples/sources/readers/files_demo \
 - **Full documentation:** https://streamwright.web.app/docs/
 - **Architecture Guide:** https://streamwright.web.app/docs/architecture/
 - **Connectors & Readers:** https://streamwright.web.app/docs/connectors/
+- **MCP Server Guide:** https://streamwright.web.app/docs/ai/mcp/

{streamwright-0.1.6 → streamwright-0.1.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.6"
+version = "0.1.7"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = {text = "Apache-2.0"}
@@ -36,13 +36,15 @@ dependencies = [
 dlt = ["dlt>=1.0"]
 
 [project.urls]
-Homepage = "https://github.com/karthick-jaganathan/streamwright"
-Repository = "https://github.com/karthick-jaganathan/streamwright"
+Homepage = "https://streamwright.web.app"
 Documentation = "https://streamwright.web.app/docs/"
+Repository = "https://github.com/karthick-jaganathan/streamwright"
+"Issue Tracker" = "https://github.com/karthick-jaganathan/streamwright/issues"
 
 [project.scripts]
 # `streamwright run`, `streamwright validate`, `streamwright connectors` - one CLI (see streamwright.core.cli)
 streamwright = "streamwright.core.cli:main"
+sw = "streamwright.core.cli:main"
 
 
 [tool.setuptools]

{streamwright-0.1.6 → streamwright-0.1.7}/src/streamwright/core/cli.py

@@ -16,7 +16,7 @@
 # **************************************************************************/
 
 """
-The `streamwright` command.
+The `streamwright` command (shorthand: `sw`).
 
   streamwright run SOURCE [--config FILE] [--set NAME=VALUE] [--secrets FILE] [--state FILE] [--stream NAME] [--output ...]
             [--file-name TEMPLATE] [--allow-connector NAME] [--summary FILE] [logging options]
@@ -25,6 +25,7 @@ The `streamwright` command.
   streamwright connectors list | search WORDS | install KEY [KEY ...] [--yes] [--hub-url URL]
                                        (the catalog; install resolves every KEY, then runs one pip install)
   streamwright validate PATH...        (streamwright-validate, plus the checks of the installed connectors and query builders)
+  streamwright mcp [--transport stdio|sse] (Model Context Protocol server for AI assistants)
 
 Logging options (streamwright run and streamwright validate): --log-level LEVEL, --log NAME=LEVEL, --log-format text|json,
 --log-config FILE, --log-max-chars N. Loggers: streamwright.source (progress, summaries, warnings), streamwright.network (INFO: a
@@ -254,6 +255,9 @@ def _parser():
     validate = commands.add_parser("validate", help="check configuration files: streamwright-validate, plus the checks of "
                                                      "the installed connectors and query builders", add_help=False)
     validate.add_argument("arguments", nargs=argparse.REMAINDER)
+
+    mcp_cmd = commands.add_parser("mcp", help="run the StreamWright Model Context Protocol (MCP) server", add_help=False)
+    mcp_cmd.add_argument("arguments", nargs=argparse.REMAINDER)
     return parser
 
 
@@ -601,6 +605,13 @@ def main(argv=None):
                                non_strict_codes=("connector-not-installed",))
         finally:
             setup.close()
+    if argv[:1] == ["mcp"]:
+        try:
+            from streamwright_mcp.cli import main as mcp_main
+        except ImportError:
+            sys.stderr.write("streamwright: 'streamwright-mcp' is not installed. Install it with: pip install streamwright-mcp\n")
+            return 1
+        return mcp_main(argv[1:]) or 0
     args = _parser().parse_args(argv)
     if args.command == "connectors":
         return _connectors_command(args)

{streamwright-0.1.6 → streamwright-0.1.7}/src/streamwright/core/validation/schema.py

@@ -16,12 +16,101 @@
 # **************************************************************************/
 
 """
-Specification of the StreamWright source configuration format (`kind: source`).
+Specification and single source of truth for the StreamWright source and stream declarative format (`kind: source`).
 
-Single source of truth for streamwright-validate's source checks (source_validator) and the published
-JSON Schemas (`streamwright-validate --export-schema DIR` writes DIR/source.schema.json and DIR/stream.schema.json for the
-stream files of source folders).
-Design: docs/design/source-format.md.
+Defines structural schemas, valid keys, data types, authentication schemes, transformation contracts,
+and export definitions for streamwright-validate, streamwright-mcp, and AI runtime assistants.
+
+### 1. `source.yaml` Schema Structure
+```yaml
+kind: source                     # Required: must be 'source'
+name: my_pipeline                # Required: snake_case identifier
+description: "Pipeline purpose"  # Optional: text description
+
+spec:                            # Declares runtime parameters and credentials
+  config:                        # Visible parameters passed via CLI (--set) or environment
+    tenant_id: {type: string, required: true}
+    batch_limit: {type: integer, default: 500}
+    start_date: {type: date, required: false}
+  secrets:                       # Redacted credentials from STREAMWRIGHT_SECRET_<NAME> or --secrets
+    api_key: {type: string, required: true}
+
+auth:                            # Authentication provider or protocol
+  provider: <connector_name>     # e.g. restapi, postgres, s3, google_ads
+  # Standard HTTP Auth Types:
+  # - bearer: {token: "{{ secrets.token }}"}
+  # - api_key: {name: "X-Api-Key", value: "{{ secrets.key }}", in: "header" | "query"}
+  # - basic: {username: "user", password: "{{ secrets.password }}"}
+  # - oauth2_refresh_token: {token_url, client_id, client_secret, refresh_token}
+
+http:                            # Optional HTTP defaults shared by all streams
+  base_url: "https://api.example.com"
+  headers: {Accept: "application/json"}
+  params_encoding: plain         # "plain" | "dotted"
+  paginator:                     # "offset" | "cursor" | "page"
+    type: cursor
+    token_path: next_cursor
+    param: cursor
+  records: {path: data}          # JMESPath/key to extract record array from envelope
+  rate_limit: {requests: 10, per: 1s}
+  retry: {codes: [429, 500, 502, 503, 504], max_attempts: 5, backoff: exponential}
+
+# Streams can be declared inline under `streams:` or as individual YAML files in `streams/<name>.yaml`.
+```
+
+### 2. `streams/<name>.yaml` Schema Structure
+```yaml
+description: "Stream documentation"
+
+# Optional partitioning (parallel or sequential chunk execution)
+# partitions:
+#   - {name: account_id, values: "{{ config.account_ids }}"}
+#   - {name: child_id, from_stream: parent_stream, field: id}
+
+# Optional incremental state sync
+# incremental:
+#   cursor_field: updated_at
+#   start: "2026-01-01T00:00:00Z"
+#   window: 1d
+#   lookback: 1h
+
+requests:                        # 1 or more extraction requests
+  # HTTP extraction:
+  - name: raw_items
+    http:
+      path: /v1/items
+      method: GET
+      params: {status: active}
+
+  # Or Connector SDK extraction:
+  # - name: raw_records
+  #   sdk: postgres
+  #   service: database
+  #   method: query
+  #   arguments:
+  #     query: "SELECT * FROM orders WHERE created_at >= $since"
+  #     params: {since: "{{ window.start }}"}
+
+transform:                       # Relational DuckDB SQL transformations
+  mode: page                     # "page" (streamed per page) or "run" (accumulated whole run)
+  steps:
+    - name: cleaned_items
+      select: |
+        SELECT 
+          record->>'id' AS item_id,
+          record->>'name' AS item_name,
+          (record->>'price')::DOUBLE AS price
+        FROM raw_items
+
+export:                          # Final output datasets and identity contracts
+  items:
+    step: cleaned_items
+    primary_key: [item_id]
+```
+
+### 3. Programmatic Schemas
+Use `json_schema()` and `stream_json_schema()` to get JSON Schema Draft 7 specifications
+for validating source and stream configurations programmatically.
 """
 
 __all__ = ["VERSION", "SUPPORTED_VERSIONS", "KIND", "json_schema", "stream_json_schema", "split_outside_quotes"]

{streamwright-0.1.6 → streamwright-0.1.7/src/streamwright.egg-info}/PKG-INFO

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.6
+Version: 0.1.7
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
 License: Apache-2.0
-Project-URL: Homepage, https://github.com/karthick-jaganathan/streamwright
-Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright
+Project-URL: Homepage, https://streamwright.web.app
 Project-URL: Documentation, https://streamwright.web.app/docs/
+Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright
+Project-URL: Issue Tracker, https://github.com/karthick-jaganathan/streamwright/issues
 Keywords: data,pipeline,etl,connector,singer,yaml,duckdb,rest-api
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
@@ -44,9 +45,12 @@ Each stream follows a three-stage pipeline:
 
 ## CLI Commands
 
-- **`streamwright run`** — Execute a source and stream records to the configured destination.
-- **`streamwright validate`** — Validate source definitions, authentication specifications, and schemas without executing network requests.
-- **`streamwright connectors`** — Browse, inspect, and install connectors from the StreamWright Hub.
+All commands can be invoked interchangeably using `streamwright` or the shorthand `sw` alias (e.g. `sw run`, `sw validate`, `sw connectors`, `sw mcp`).
+
+- **`streamwright run`** (or **`sw run`**) — Execute a source and stream records to the configured destination.
+- **`streamwright validate`** (or **`sw validate`**) — Validate source definitions, authentication specifications, and schemas without executing network requests.
+- **`streamwright connectors`** (or **`sw connectors`**) — Browse, inspect, and install connectors from the StreamWright Hub.
+- **`streamwright mcp`** (or **`sw mcp`**) — Launch the Model Context Protocol (MCP) server for AI coding assistants.
 
 ## Source Structure
 
@@ -120,3 +124,4 @@ streamwright run streamwright/examples/sources/readers/files_demo \
 - **Full documentation:** https://streamwright.web.app/docs/
 - **Architecture Guide:** https://streamwright.web.app/docs/architecture/
 - **Connectors & Readers:** https://streamwright.web.app/docs/connectors/
+- **MCP Server Guide:** https://streamwright.web.app/docs/ai/mcp/

{streamwright-0.1.6 → streamwright-0.1.7}/src/streamwright.egg-info/SOURCES.txt

@@ -40,6 +40,7 @@ src/streamwright/core/validation/engine.py
 src/streamwright/core/validation/schema.py
 src/streamwright/core/validation/source.py
 tests/test_catalog.py
+tests/test_cli_subcommands.py
 tests/test_connectors.py
 tests/test_dlt_output.py
 tests/test_exact_decimals.py

{streamwright-0.1.6 → streamwright-0.1.7}/src/streamwright.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 streamwright = streamwright.core.cli:main
+sw = streamwright.core.cli:main

streamwright-0.1.7/tests/test_cli_subcommands.py

@@ -0,0 +1,32 @@
+import subprocess
+import sys
+from unittest.mock import patch
+
+from streamwright.core import cli
+
+
+def test_cli_help():
+    result = subprocess.run([sys.executable, "-m", "streamwright.core.cli", "--help"],
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                            universal_newlines=True)
+    assert result.returncode == 0
+    assert "mcp" in result.stdout
+
+
+def test_cli_mcp_delegation():
+    from unittest.mock import MagicMock
+    mock_mcp = MagicMock(return_value=0)
+    mock_mod = MagicMock(main=mock_mcp)
+    with patch.dict(sys.modules, {"streamwright_mcp": MagicMock(), "streamwright_mcp.cli": mock_mod}):
+        code = cli.main(["mcp", "--help"])
+        assert code == 0
+        mock_mcp.assert_called_once_with(["--help"])
+
+
+def test_cli_mcp_not_installed(capsys):
+    with patch.dict(sys.modules, {"streamwright_mcp.cli": None}):
+        with patch("builtins.__import__", side_effect=ImportError("No module named 'streamwright_mcp'")):
+            code = cli.main(["mcp"])
+            assert code == 1
+            captured = capsys.readouterr()
+            assert "'streamwright-mcp' is not installed" in captured.err

{streamwright-0.1.6 → streamwright-0.1.7}/tests/test_validation.py

@@ -376,7 +376,15 @@ def test_templates_are_not_allowed_in_names_and_records():
 # * ----
 
 def test_auth_checks():
-    assert codes(check(lambda d: d["auth"].update(provider="google_ads"))) == [(ERROR, "bad-value")]
+    from streamwright.core.runtime.components import Connector, register, unregister
+    class MockSdkValConnector(Connector):
+        name = "mock_sdk_val"
+        transport = "sdk"
+    register(MockSdkValConnector())
+    try:
+        assert codes(check(lambda d: d["auth"].update(provider="mock_sdk_val"))) == [(ERROR, "bad-value")]
+    finally:
+        unregister("mock_sdk_val")
     assert codes(check(lambda d: d["auth"].update(type="bearr"))) == [(ERROR, "bad-value")]
     issues = check(lambda d: d.update(auth={"type": "oauth2_refresh_token", "token_url": "https://x/token",
                                             "client_id": "id", "client_secret": "s3cret",