streamwright 0.1.8__tar.gz → 0.1.9__tar.gz

streamwright-0.1.9/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: streamwright
+Version: 0.1.9
+Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
+Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://streamwright.web.app
+Project-URL: Documentation, https://streamwright.web.app/docs/
+Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright
+Project-URL: Issue Tracker, https://github.com/karthick-jaganathan/streamwright/issues
+Keywords: data,pipeline,etl,connector,singer,yaml,duckdb,rest-api
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml==6.0.2
+Requires-Dist: requests<3,>=2.25
+Requires-Dist: duckdb<2.1,>=2.0
+Requires-Dist: pytz
+Provides-Extra: dlt
+Requires-Dist: dlt>=1.0; extra == "dlt"
+Dynamic: license-file
+
+# StreamWright
+
+[![PyPI](https://img.shields.io/pypi/v/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![Python](https://img.shields.io/pypi/pyversions/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![License](https://img.shields.io/pypi/l/streamwright.svg)](https://github.com/karthick-jaganathan/streamwright/blob/master/LICENSE)
+[![Docs](https://img.shields.io/badge/docs-streamwright.web.app-brightgreen.svg)](https://streamwright.web.app/docs/)
+
+Declarative data pipelines in YAML: read from APIs, files, databases and ads platforms, shape with DuckDB SQL,
+write to files, DuckDB, DuckLake, Singer or dlt.
+
+- **No per-connector code**: authentication, paging, retries, rate limits and incremental state are configuration.
+- **Connectors on demand** from the StreamWright hub: `sw connectors -i <name>`.
+- **Jobs**: `kind: DuckDBJob` SQL pipelines with replay-safe writes; `kind: SparkJob` with
+  [streamwright-spark](https://pypi.org/project/streamwright-spark/).
+
+## Install
+
+```bash
+pip install streamwright
+sw connectors -i restapi
+```
+
+## Example
+
+A source is a folder: `source.yaml` and one file per stream.
+
+```yaml
+# my_api/source.yaml
+kind: source
+name: my_api
+spec:
+  secrets:
+    api_token: {type: string}
+auth:
+  provider: restapi
+  type: bearer
+  token: "{{ secrets.api_token }}"
+http:
+  base_url: https://api.example.com
+```
+
+```yaml
+# my_api/streams/items.yaml
+requests:
+  - name: raw_items
+    http: {path: /v1/items}
+transform:
+  mode: page
+  steps:
+    - name: items
+      select: SELECT record->>'id' AS id, record->>'name' AS name FROM raw_items
+export:
+  items: {step: items, primary_key: [id]}
+```
+
+```bash
+sw validate my_api
+STREAMWRIGHT_SECRET_API_TOKEN=... sw run my_api --output jsonl:out
+```
+
+## Links
+
+[Documentation](https://streamwright.web.app/docs/) ·
+[Quickstart](https://streamwright.web.app/docs/quickstart/) ·
+[Source code](https://github.com/karthick-jaganathan/streamwright)

streamwright-0.1.9/README.md

@@ -0,0 +1,65 @@
+# StreamWright
+
+[![PyPI](https://img.shields.io/pypi/v/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![Python](https://img.shields.io/pypi/pyversions/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![License](https://img.shields.io/pypi/l/streamwright.svg)](https://github.com/karthick-jaganathan/streamwright/blob/master/LICENSE)
+[![Docs](https://img.shields.io/badge/docs-streamwright.web.app-brightgreen.svg)](https://streamwright.web.app/docs/)
+
+Declarative data pipelines in YAML: read from APIs, files, databases and ads platforms, shape with DuckDB SQL,
+write to files, DuckDB, DuckLake, Singer or dlt.
+
+- **No per-connector code**: authentication, paging, retries, rate limits and incremental state are configuration.
+- **Connectors on demand** from the StreamWright hub: `sw connectors -i <name>`.
+- **Jobs**: `kind: DuckDBJob` SQL pipelines with replay-safe writes; `kind: SparkJob` with
+  [streamwright-spark](https://pypi.org/project/streamwright-spark/).
+
+## Install
+
+```bash
+pip install streamwright
+sw connectors -i restapi
+```
+
+## Example
+
+A source is a folder: `source.yaml` and one file per stream.
+
+```yaml
+# my_api/source.yaml
+kind: source
+name: my_api
+spec:
+  secrets:
+    api_token: {type: string}
+auth:
+  provider: restapi
+  type: bearer
+  token: "{{ secrets.api_token }}"
+http:
+  base_url: https://api.example.com
+```
+
+```yaml
+# my_api/streams/items.yaml
+requests:
+  - name: raw_items
+    http: {path: /v1/items}
+transform:
+  mode: page
+  steps:
+    - name: items
+      select: SELECT record->>'id' AS id, record->>'name' AS name FROM raw_items
+export:
+  items: {step: items, primary_key: [id]}
+```
+
+```bash
+sw validate my_api
+STREAMWRIGHT_SECRET_API_TOKEN=... sw run my_api --output jsonl:out
+```
+
+## Links
+
+[Documentation](https://streamwright.web.app/docs/) ·
+[Quickstart](https://streamwright.web.app/docs/quickstart/) ·
+[Source code](https://github.com/karthick-jaganathan/streamwright)

{streamwright-0.1.8 → streamwright-0.1.9}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.8"
+version = "0.1.9"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = {text = "Apache-2.0"}
@@ -16,18 +16,17 @@ classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "pyyaml==6.0.2",
     "requests>=2.25,<3",
     # `select`: the SQL that shapes each stream's records runs in an embedded DuckDB (which needs pytz for
     # timestamps with time zones)
-    "duckdb>=1.5.2",
+    "duckdb>=2.0,<2.1",
     "pytz",
 ]
 
@@ -46,17 +45,28 @@ Repository = "https://github.com/karthick-jaganathan/streamwright"
 streamwright = "streamwright.core.cli:main"
 sw = "streamwright.core.cli:main"
 
+[project.entry-points."streamwright.commands"]
+duckdb = "streamwright.core.cli.commands.jobs:register"
+
+[project.entry-points."streamwright.job_kinds"]
+DuckDBJob = "streamwright.core.jobs.duckdb.plugin:JOB_ENGINE"
+
+[project.entry-points."streamwright.duckdb.connectors"]
+# the DuckDBJob engine's built-in adapters; every other provider's adapter ships with its connector
+duckdb = "streamwright.core.jobs.duckdb.adapter:DuckDBCatalogAdapter"
+
 
 [tool.setuptools]
 # src layout: `streamwright` is a namespace package shared with the connector distributions (streamwright.connectors.*)
 package-dir = {"" = "src"}
-packages = [
-    "streamwright.core",
-    "streamwright.core.engine",
-    "streamwright.core.outputs",
-    "streamwright.core.net",
-    "streamwright.core.config",
-    "streamwright.core.validation",
-    "streamwright.core.runtime",
-]
 
+[tool.setuptools.packages.find]
+# every streamwright.core subpackage ships (jobs, sql, ...); `streamwright` itself stays a namespace
+where = ["src"]
+include = ["streamwright.core*"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"streamwright.core" = ["py.typed"]
+"streamwright.core.sdk" = ["py.typed"]
+"streamwright.core.jobs.duckdb" = ["guide.md"]

{streamwright-0.1.8 → streamwright-0.1.9}/src/streamwright/core/catalog.py

@@ -12,12 +12,15 @@ what users type.
 Security: non-"official" connectors run third-party code, so `install` asks for
 confirmation (or `--yes`) and refuses non-interactive installs without it.
 """
+import datetime
 import json
 import os
 import subprocess
 import sys
 from pathlib import Path
 
+import requests
+
 from streamwright.core.runtime import components
 
 HUB_ENV = "STREAMWRIGHT_HUB_URL"
@@ -26,13 +29,17 @@ CACHE = Path(os.path.expanduser("~/.streamwright/connectors.json"))
 TIMEOUT_S = 5
 
 
+class _Catalog(dict):
+    def __init__(self, data, trusted_origin=False):
+        super().__init__(data)
+        self.trusted_origin = trusted_origin
+
+
 def _valid(data):
     return isinstance(data, dict) and isinstance(data.get("connectors"), dict)
 
 
 def _remote(url):
-    import requests
-
     response = requests.get(url, timeout=TIMEOUT_S)
     response.raise_for_status()
     data = response.json()
@@ -53,7 +60,8 @@ def load(hub_url=None):
     """
     url = hub_url or os.environ.get(HUB_ENV) or DEFAULT_HUB
     try:
-        return _remote(url)
+        data = _remote(url)
+        return _Catalog(data, trusted_origin=(url == DEFAULT_HUB and url.startswith("https://")))
     except Exception as exc:  # offline, hub down or a bad publish: fall back, visibly
         reason = type(exc).__name__  # network errors: the type is enough; an HTTP status or a bad index says why
         status = getattr(getattr(exc, "response", None), "status_code", None)
@@ -65,8 +73,8 @@ def load(hub_url=None):
         try:
             data = json.loads(CACHE.read_text())
             if _valid(data):
-                sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the copy fetched on %s\n"
-                                 % (url, reason, _mtime(CACHE)))
+                sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the copy fetched on "
+                                 "%s\n" % (url, reason, _mtime(CACHE)))
                 return data
         except Exception:
             pass
@@ -74,12 +82,12 @@ def load(hub_url=None):
 
 
 def _mtime(path):
-    import datetime
     return datetime.datetime.fromtimestamp(path.stat().st_mtime).strftime("%Y-%m-%d %H:%M")
 
 
 def connectors(hub_url=None):
-    return load(hub_url).get("connectors", {})
+    data = load(hub_url)
+    return _Catalog(data.get("connectors", {}), getattr(data, "trusted_origin", False))
 
 
 def installed():
@@ -116,13 +124,14 @@ def listing(hub_url=None):
     have = installed()
     lines = []
     by_family = {}
-    for key, entry in connectors(hub_url).items():
+    catalog_data = connectors(hub_url)
+    for key, entry in catalog_data.items():
         by_family.setdefault(entry.get("family", "other"), []).append((key, entry))
     for family in sorted(by_family):
         lines.append("%s:" % family)
         for key, entry in sorted(by_family[family]):
             mark = "[installed]" if key in have else "          "
-            trust = entry.get("trust", "community")
+            trust = entry.get("trust", "community") if catalog_data.trusted_origin else "community"
             lines.append("  %s %-16s %-10s %s" % (mark, key, trust, entry.get("summary", "")))
     return lines
 
@@ -144,6 +153,7 @@ def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
         sys.stderr.write("streamwright: 'connectors install' needs at least one connector KEY\n")
         return 2
     catalog = connectors(hub_url)
+    trusted_origin = getattr(catalog, "trusted_origin", False)
     unknown = [key for key in keys if key not in catalog]
     if unknown:
         sys.stderr.write("streamwright: unknown connector%s %s (try: streamwright connectors list)\n" % (
@@ -163,8 +173,11 @@ def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
         if not spec:
             sys.stderr.write("streamwright: connector %r has no installable source\n" % key)
             return 2
+        if not isinstance(spec, str) or spec.lstrip().startswith("-"):
+            sys.stderr.write("streamwright: connector %r has an invalid install requirement\n" % key)
+            return 2
         specs.append(spec)
-        trust = entry.get("trust", "community")
+        trust = entry.get("trust", "community") if trusted_origin else "community"
         if trust != "official":
             untrusted.append((key, trust, _source_of(inst)))
     if not specs:
@@ -184,6 +197,6 @@ def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
             print("aborted")
             return 1
 
-    cmd = [sys.executable, "-m", "pip", "install", *pip_args, *specs]
+    cmd = [sys.executable, "-m", "pip", "install", *pip_args, "--", *specs]
     print("+ " + " ".join(cmd), flush=True)
     return subprocess.call(cmd)

streamwright-0.1.9/src/streamwright/core/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Public StreamWright command-line interface."""
+
+from streamwright.core.cli.main import command_parsers, detect_kind, main, run
+
+__all__ = ["command_parsers", "detect_kind", "main", "run"]

streamwright-0.1.9/src/streamwright/core/cli/__main__.py

@@ -0,0 +1,5 @@
+"""Run the CLI as a module."""
+
+from streamwright.core.cli import main
+
+raise SystemExit(main())

streamwright-0.1.9/src/streamwright/core/cli/commands/__init__.py

@@ -0,0 +1 @@
+"""Built-in commands."""

streamwright-0.1.9/src/streamwright/core/cli/commands/connectors.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# /*************************************************************************
+# * Copyright 2025 Karthick Jaganathan
+# *
+# * Licensed under the Apache License, Version 2.0 (the "License");
+# * you may not use this file except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# * https://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# **************************************************************************/
+
+"""`streamwright connectors`: the installed connectors, and the catalog's (list, search, install)."""
+
+import sys
+
+from streamwright.core import catalog
+from streamwright.core.runtime import components, plugins
+from streamwright.core.runtime.definitions import engine_reach
+
+
+def _connector_lines():
+    """
+    The installed (and registered) connectors by family: name, title, distribution and version, and how each engine
+    reaches them — native or via a call step, in the fixed order source, duckdb, spark (only specs are imported).
+    """
+    metadata = {entry["name"]: entry for entry in plugins.discover(plugins.CONNECTORS)}
+    groups = {}
+    for name in components.available():
+        entry = metadata.get(name, {})
+        package = "%s %s" % (entry.get("distribution") or "registered in-process", entry.get("version") or "")
+        try:
+            definition = components.load(name)
+        except components.ComponentLoadError as exc:
+            groups.setdefault("unavailable", []).append("  %-15s%s" % (name, exc))
+            continue
+        groups.setdefault(definition.family, []).append("  %-15s%-23s%-34s[%s]" % (
+            name, definition.title, package.strip(), ", ".join(engine_reach(definition))))
+    lines = []
+    for family in sorted(groups):
+        if lines:
+            lines.append("")
+        lines.append("%s:" % family)
+        lines.extend(sorted(groups[family]))
+    return lines
+
+
+def run_connectors_command(args):
+    """`streamwright connectors`: bare = installed list; ls/i/find use the catalog (the hub)."""
+    if getattr(args, "flag_list", False):
+        action = "ls"
+        query = []
+    elif getattr(args, "flag_install", None):
+        action = "install"
+        query = args.flag_install
+    elif getattr(args, "flag_search", None):
+        action = "find"
+        query = args.flag_search
+    else:
+        action = getattr(args, "action", None)
+        query = getattr(args, "query", [])
+
+    if action is None:
+        lines = _connector_lines()
+        print("\n".join(lines) if lines else "no connectors installed (use streamwright connectors list to find one)")
+        return 0
+
+    norm_action = action.lower()
+    if norm_action in ("ls", "list", "-ls", "--list"):
+        lines = catalog.listing(args.hub_url)
+        print("\n".join(lines) if lines else "the catalog is empty")
+        return 0
+
+    if norm_action in ("i", "install", "-i", "--install"):
+        if not query:
+            sys.stderr.write("streamwright: 'connectors -i' needs at least one connector KEY\n")
+            return 2
+        return catalog.install_many(query, assume_yes=args.yes, hub_url=args.hub_url)
+
+    if norm_action in ("find", "search", "s", "-s", "-f", "--find", "--search"):
+        text = " ".join(query)
+        hits = catalog.search(text, args.hub_url)
+        if not hits:
+            print("no connectors match %r" % text)
+            return 0
+        for key, entry in sorted(hits.items()):
+            print("  %-16s %-10s %s" % (key, entry.get("trust", "community"), entry.get("summary", "")))
+        return 0
+
+    # If action was not a reserved verb, treat it as a direct search query (e.g. `sw connectors ads`)
+    text = " ".join([action, *query])
+    hits = catalog.search(text, args.hub_url)
+    if not hits:
+        print("no connectors match %r" % text)
+        return 0
+    for key, entry in sorted(hits.items()):
+        print("  %-16s %-10s %s" % (key, entry.get("trust", "community"), entry.get("summary", "")))
+    return 0

streamwright-0.1.9/src/streamwright/core/cli/commands/jobs.py

@@ -0,0 +1,27 @@
+"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
+
+import argparse
+import sys
+
+from streamwright.core.jobs.engine import load_job_engine
+from streamwright.core.runtime import plugins
+
+
+def register(subparsers):
+    parser = subparsers.add_parser("duckdb", help="run an installed DuckDBJob engine", add_help=False)
+    parser.add_argument("arguments", nargs=argparse.REMAINDER)
+    parser.set_defaults(command_handler=lambda args: execute(args.arguments))
+
+
+def execute(arguments):
+    return run_job_kind("DuckDBJob", arguments)
+
+
+def run_job_kind(kind, argv):
+    """Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
+    try:
+        job_engine = load_job_engine(kind)
+    except plugins.PluginError as exc:
+        sys.stderr.write("streamwright: %s\n" % exc)
+        return 2
+    return job_engine.main(argv) or 0