streamwright 0.1.8__tar.gz → 0.1.10__tar.gz

streamwright-0.1.10/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.4
+Name: streamwright
+Version: 0.1.10
+Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
+Author-email: Karthick Jaganathan <kjaganathan.sde@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://streamwright.web.app
+Project-URL: Documentation, https://streamwright.web.app/docs/
+Project-URL: Repository, https://github.com/karthick-jaganathan/streamwright
+Project-URL: Issue Tracker, https://github.com/karthick-jaganathan/streamwright/issues
+Keywords: data,pipeline,etl,connector,singer,yaml,duckdb,rest-api
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml==6.0.2
+Requires-Dist: requests<3,>=2.25
+Requires-Dist: duckdb<2.1,>=2.0
+Requires-Dist: pytz
+Provides-Extra: dlt
+Requires-Dist: dlt>=1.0; extra == "dlt"
+Dynamic: license-file
+
+# StreamWright
+
+[![PyPI](https://img.shields.io/pypi/v/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![Python](https://img.shields.io/pypi/pyversions/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![License](https://img.shields.io/pypi/l/streamwright.svg)](https://github.com/karthick-jaganathan/streamwright/blob/master/LICENSE)
+[![Docs](https://img.shields.io/badge/docs-streamwright.web.app-brightgreen.svg)](https://streamwright.web.app/docs/)
+
+Declarative data pipelines in YAML: read from APIs, files, databases and ads platforms, shape with DuckDB SQL,
+write to files, DuckDB, DuckLake, Singer or dlt.
+
+- **No per-connector code**: authentication, paging, retries, rate limits and incremental state are configuration.
+- **Connectors on demand** from the StreamWright hub: `sw connectors -i <name>`.
+- **Jobs**: `kind: DuckDBJob` SQL pipelines with replay-safe writes; `kind: SparkJob` with
+  [streamwright-spark](https://pypi.org/project/streamwright-spark/).
+
+## Install
+
+```bash
+pip install streamwright
+sw connectors -i restapi
+```
+
+## Example
+
+A source is a folder: `source.yaml` and one file per stream.
+
+```yaml
+# my_api/source.yaml
+kind: source
+name: my_api
+spec:
+  secrets:
+    api_token: {type: string}
+auth:
+  provider: restapi
+  type: bearer
+  token: "{{ secrets.api_token }}"
+http:
+  base_url: https://api.example.com
+```
+
+```yaml
+# my_api/streams/items.yaml
+requests:
+  - name: raw_items
+    http: {path: /v1/items}
+transform:
+  mode: page
+  steps:
+    - name: items
+      select: SELECT record->>'id' AS id, record->>'name' AS name FROM raw_items
+export:
+  items: {step: items, primary_key: [id]}
+```
+
+```bash
+sw validate my_api
+STREAMWRIGHT_SECRET_API_TOKEN=... sw run my_api --output jsonl:out
+```
+
+## Links
+
+[Documentation](https://streamwright.web.app/docs/) ·
+[Quickstart](https://streamwright.web.app/docs/quickstart/) ·
+[Source code](https://github.com/karthick-jaganathan/streamwright)

streamwright-0.1.10/README.md

@@ -0,0 +1,65 @@
+# StreamWright
+
+[![PyPI](https://img.shields.io/pypi/v/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![Python](https://img.shields.io/pypi/pyversions/streamwright.svg)](https://pypi.org/project/streamwright/)
+[![License](https://img.shields.io/pypi/l/streamwright.svg)](https://github.com/karthick-jaganathan/streamwright/blob/master/LICENSE)
+[![Docs](https://img.shields.io/badge/docs-streamwright.web.app-brightgreen.svg)](https://streamwright.web.app/docs/)
+
+Declarative data pipelines in YAML: read from APIs, files, databases and ads platforms, shape with DuckDB SQL,
+write to files, DuckDB, DuckLake, Singer or dlt.
+
+- **No per-connector code**: authentication, paging, retries, rate limits and incremental state are configuration.
+- **Connectors on demand** from the StreamWright hub: `sw connectors -i <name>`.
+- **Jobs**: `kind: DuckDBJob` SQL pipelines with replay-safe writes; `kind: SparkJob` with
+  [streamwright-spark](https://pypi.org/project/streamwright-spark/).
+
+## Install
+
+```bash
+pip install streamwright
+sw connectors -i restapi
+```
+
+## Example
+
+A source is a folder: `source.yaml` and one file per stream.
+
+```yaml
+# my_api/source.yaml
+kind: source
+name: my_api
+spec:
+  secrets:
+    api_token: {type: string}
+auth:
+  provider: restapi
+  type: bearer
+  token: "{{ secrets.api_token }}"
+http:
+  base_url: https://api.example.com
+```
+
+```yaml
+# my_api/streams/items.yaml
+requests:
+  - name: raw_items
+    http: {path: /v1/items}
+transform:
+  mode: page
+  steps:
+    - name: items
+      select: SELECT record->>'id' AS id, record->>'name' AS name FROM raw_items
+export:
+  items: {step: items, primary_key: [id]}
+```
+
+```bash
+sw validate my_api
+STREAMWRIGHT_SECRET_API_TOKEN=... sw run my_api --output jsonl:out
+```
+
+## Links
+
+[Documentation](https://streamwright.web.app/docs/) ·
+[Quickstart](https://streamwright.web.app/docs/quickstart/) ·
+[Source code](https://github.com/karthick-jaganathan/streamwright)

{streamwright-0.1.8 → streamwright-0.1.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.8"
+version = "0.1.10"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = {text = "Apache-2.0"}
@@ -16,18 +16,17 @@ classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "pyyaml==6.0.2",
     "requests>=2.25,<3",
     # `select`: the SQL that shapes each stream's records runs in an embedded DuckDB (which needs pytz for
     # timestamps with time zones)
-    "duckdb>=1.5.2",
+    "duckdb>=2.0,<2.1",
     "pytz",
 ]
 
@@ -46,17 +45,28 @@ Repository = "https://github.com/karthick-jaganathan/streamwright"
 streamwright = "streamwright.core.cli:main"
 sw = "streamwright.core.cli:main"
 
+[project.entry-points."streamwright.commands"]
+duckdb = "streamwright.core.cli.commands.jobs:register"
+
+[project.entry-points."streamwright.job_kinds"]
+DuckDBJob = "streamwright.core.jobs.duckdb.plugin:JOB_ENGINE"
+
+[project.entry-points."streamwright.duckdb.connectors"]
+# the DuckDBJob engine's built-in adapters; every other provider's adapter ships with its connector
+duckdb = "streamwright.core.jobs.duckdb.adapter:DuckDBCatalogAdapter"
+
 
 [tool.setuptools]
 # src layout: `streamwright` is a namespace package shared with the connector distributions (streamwright.connectors.*)
 package-dir = {"" = "src"}
-packages = [
-    "streamwright.core",
-    "streamwright.core.engine",
-    "streamwright.core.outputs",
-    "streamwright.core.net",
-    "streamwright.core.config",
-    "streamwright.core.validation",
-    "streamwright.core.runtime",
-]
 
+[tool.setuptools.packages.find]
+# every streamwright.core subpackage ships (jobs, sql, ...); `streamwright` itself stays a namespace
+where = ["src"]
+include = ["streamwright.core*"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"streamwright.core" = ["py.typed"]
+"streamwright.core.sdk" = ["py.typed"]
+"streamwright.core.jobs.duckdb" = ["guide.md"]

{streamwright-0.1.8 → streamwright-0.1.10}/src/streamwright/core/catalog.py

@@ -12,12 +12,15 @@ what users type.
 Security: non-"official" connectors run third-party code, so `install` asks for
 confirmation (or `--yes`) and refuses non-interactive installs without it.
 """
+import datetime
 import json
 import os
 import subprocess
 import sys
 from pathlib import Path
 
+import requests
+
 from streamwright.core.runtime import components
 
 HUB_ENV = "STREAMWRIGHT_HUB_URL"
@@ -26,13 +29,17 @@ CACHE = Path(os.path.expanduser("~/.streamwright/connectors.json"))
 TIMEOUT_S = 5
 
 
+class _Catalog(dict):
+    def __init__(self, data, trusted_origin=False):
+        super().__init__(data)
+        self.trusted_origin = trusted_origin
+
+
 def _valid(data):
     return isinstance(data, dict) and isinstance(data.get("connectors"), dict)
 
 
 def _remote(url):
-    import requests
-
     response = requests.get(url, timeout=TIMEOUT_S)
     response.raise_for_status()
     data = response.json()
@@ -53,7 +60,8 @@ def load(hub_url=None):
     """
     url = hub_url or os.environ.get(HUB_ENV) or DEFAULT_HUB
     try:
-        return _remote(url)
+        data = _remote(url)
+        return _Catalog(data, trusted_origin=(url == DEFAULT_HUB and url.startswith("https://")))
     except Exception as exc:  # offline, hub down or a bad publish: fall back, visibly
         reason = type(exc).__name__  # network errors: the type is enough; an HTTP status or a bad index says why
         status = getattr(getattr(exc, "response", None), "status_code", None)
@@ -65,8 +73,8 @@ def load(hub_url=None):
         try:
             data = json.loads(CACHE.read_text())
             if _valid(data):
-                sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the copy fetched on %s\n"
-                                 % (url, reason, _mtime(CACHE)))
+                sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the copy fetched on "
+                                 "%s\n" % (url, reason, _mtime(CACHE)))
                 return data
         except Exception:
             pass
@@ -74,12 +82,12 @@ def load(hub_url=None):
 
 
 def _mtime(path):
-    import datetime
     return datetime.datetime.fromtimestamp(path.stat().st_mtime).strftime("%Y-%m-%d %H:%M")
 
 
 def connectors(hub_url=None):
-    return load(hub_url).get("connectors", {})
+    data = load(hub_url)
+    return _Catalog(data.get("connectors", {}), getattr(data, "trusted_origin", False))
 
 
 def installed():
@@ -116,13 +124,14 @@ def listing(hub_url=None):
     have = installed()
     lines = []
     by_family = {}
-    for key, entry in connectors(hub_url).items():
+    catalog_data = connectors(hub_url)
+    for key, entry in catalog_data.items():
         by_family.setdefault(entry.get("family", "other"), []).append((key, entry))
     for family in sorted(by_family):
         lines.append("%s:" % family)
         for key, entry in sorted(by_family[family]):
             mark = "[installed]" if key in have else "          "
-            trust = entry.get("trust", "community")
+            trust = entry.get("trust", "community") if catalog_data.trusted_origin else "community"
             lines.append("  %s %-16s %-10s %s" % (mark, key, trust, entry.get("summary", "")))
     return lines
 
@@ -144,6 +153,7 @@ def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
         sys.stderr.write("streamwright: 'connectors install' needs at least one connector KEY\n")
         return 2
     catalog = connectors(hub_url)
+    trusted_origin = getattr(catalog, "trusted_origin", False)
     unknown = [key for key in keys if key not in catalog]
     if unknown:
         sys.stderr.write("streamwright: unknown connector%s %s (try: streamwright connectors list)\n" % (
@@ -163,8 +173,11 @@ def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
         if not spec:
             sys.stderr.write("streamwright: connector %r has no installable source\n" % key)
             return 2
+        if not isinstance(spec, str) or spec.lstrip().startswith("-"):
+            sys.stderr.write("streamwright: connector %r has an invalid install requirement\n" % key)
+            return 2
         specs.append(spec)
-        trust = entry.get("trust", "community")
+        trust = entry.get("trust", "community") if trusted_origin else "community"
         if trust != "official":
             untrusted.append((key, trust, _source_of(inst)))
     if not specs:
@@ -184,6 +197,6 @@ def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
             print("aborted")
             return 1
 
-    cmd = [sys.executable, "-m", "pip", "install", *pip_args, *specs]
+    cmd = [sys.executable, "-m", "pip", "install", *pip_args, "--", *specs]
     print("+ " + " ".join(cmd), flush=True)
     return subprocess.call(cmd)

streamwright-0.1.10/src/streamwright/core/cli/__init__.py

@@ -0,0 +1,5 @@
+"""Public StreamWright command-line interface."""
+
+from streamwright.core.cli.main import command_parsers, detect_kind, main, run
+
+__all__ = ["command_parsers", "detect_kind", "main", "run"]

streamwright-0.1.10/src/streamwright/core/cli/__main__.py

@@ -0,0 +1,5 @@
+"""Run the CLI as a module."""
+
+from streamwright.core.cli import main
+
+raise SystemExit(main())

streamwright-0.1.10/src/streamwright/core/cli/commands/__init__.py

@@ -0,0 +1 @@
+"""Built-in commands."""

streamwright-0.1.10/src/streamwright/core/cli/commands/connectors.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# /*************************************************************************
+# * Copyright 2025 Karthick Jaganathan
+# *
+# * Licensed under the Apache License, Version 2.0 (the "License");
+# * you may not use this file except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# * https://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# **************************************************************************/
+
+"""`streamwright connectors`: the installed connectors, and the catalog's (list, search, install)."""
+
+import re
+import sys
+from importlib import metadata
+
+import duckdb
+
+from streamwright.core import catalog
+from streamwright.core.runtime import components, plugins
+from streamwright.core.runtime.logs.lines import NETWORK
+
+_REQUIREMENT_NAME = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
+# a connector that logs on streamwright.network calls through StreamWright's HTTP client, which is built on requests
+HTTP_CLIENT_LIBRARY = "requests"
+
+
+def _installed(name):
+    try:
+        return "%s==%s" % (name, metadata.version(name))
+    except metadata.PackageNotFoundError:
+        return "%s (not installed)" % name
+
+
+def libraries(distribution, loggers=()):
+    """
+    The third-party libraries a connector uses, as "name==version" (or "name (not installed)"): what its distribution
+    requires (not its extras', not StreamWright's), and requests when it calls through StreamWright's HTTP client.
+    """
+    try:
+        requires = (metadata.requires(distribution) or []) if distribution else []
+    except metadata.PackageNotFoundError:
+        requires = []
+    names = []
+    for requirement in requires:
+        spec, _, marker = requirement.partition(";")
+        match = _REQUIREMENT_NAME.match(spec)
+        if "extra" not in marker and match and not match.group(1).lower().startswith("streamwright"):
+            names.append(match.group(1))
+    if NETWORK.name in loggers and HTTP_CLIENT_LIBRARY not in names:
+        names.append(HTTP_CLIENT_LIBRARY)
+    return [_installed(name) for name in names]
+
+
+class DuckDBExtensions:
+    """The installed DuckDB's extensions (one in-memory query, made when a connector names one): name@version."""
+
+    def __init__(self):
+        self._installed = None
+
+    def _load(self):
+        self._installed = {}
+        rows = duckdb.connect().execute(
+            "SELECT extension_name, aliases, installed, extension_version FROM duckdb_extensions()").fetchall()
+        for name, aliases, installed, version in rows:
+            for key in (name, *(aliases or ())):
+                self._installed[key] = (name, version if installed else None)
+        self._installed[None] = ("duckdb", duckdb.__version__)
+
+    def describe(self, names):
+        """("duckdb==VERSION", ["EXTENSION@VERSION" or "EXTENSION (not installed)", ...]); (None, []) for no names."""
+        if not names:
+            return None, []
+        if self._installed is None:
+            self._load()
+        found = []
+        for name in names:
+            extension, version = self._installed.get(name, (name, None))
+            found.append("%s@%s" % (extension, version) if version else "%s (not installed)" % extension)
+        return "duckdb==%s" % self._installed[None][1], found
+
+
+def _fields(name, definition, entry, extensions):
+    """A connector's `pip show`-style fields."""
+    used = libraries(entry.get("distribution"), definition.loggers)
+    duckdb_library, loaded = extensions.describe(definition.duckdb_extensions)
+    return (("Name", name), ("Title", definition.title), ("Family", definition.family),
+            ("Summary", definition.summary), ("Package", entry.get("distribution") or "(registered in-process)"),
+            ("Version", entry.get("version") or ""), ("Libraries", ", ".join(used + [duckdb_library] * bool(loaded))),
+            ("DuckDB-Extensions", ", ".join(loaded)), ("Loggers", ", ".join(definition.loggers)))
+
+
+def _connector_lines(names=None):
+    """
+    The installed (and registered) connectors, sorted by family and name, as `pip show` shows packages (`---` between
+    them): name, title, family, summary, distribution and version, the third-party libraries and DuckDB extensions it
+    uses (installed versions) and its loggers (for --log). `names`: only these. Only specs are imported.
+    """
+    metadata_of = {entry["name"]: entry for entry in plugins.discover(plugins.CONNECTORS)}
+    blocks, extensions = [], DuckDBExtensions()
+    for name in components.available():
+        if names is not None and name not in names:
+            continue
+        try:
+            definition = components.load(name)
+        except components.ComponentLoadError as exc:
+            blocks.append(("~", name, (("Name", name), ("Error", str(exc)))))  # (after the families)
+            continue
+        blocks.append((definition.family, name, _fields(name, definition, metadata_of.get(name, {}), extensions)))
+    lines = []
+    for _, _, fields in sorted(blocks, key=lambda block: block[:2]):
+        if lines:
+            lines.append("---")
+        lines.extend(("%s: %s" % (key, value)).rstrip() for key, value in fields)
+    return lines
+
+
+def run_connectors_command(args):
+    """`streamwright connectors`: bare or installed names = installed connectors; ls/i/find use the catalog (hub)."""
+    if getattr(args, "flag_list", False):
+        action = "ls"
+        query = []
+    elif getattr(args, "flag_install", None):
+        action = "install"
+        query = args.flag_install
+    elif getattr(args, "flag_search", None):
+        action = "find"
+        query = args.flag_search
+    else:
+        action = getattr(args, "action", None)
+        query = getattr(args, "query", [])
+
+    if action is None:
+        lines = _connector_lines()
+        print("\n".join(lines) if lines else "no connectors installed (use streamwright connectors list to find one)")
+        return 0
+
+    norm_action = action.lower()
+    if norm_action in ("ls", "list", "-ls", "--list"):
+        lines = catalog.listing(args.hub_url)
+        print("\n".join(lines) if lines else "the catalog is empty")
+        return 0
+
+    if norm_action in ("i", "install", "-i", "--install"):
+        if not query:
+            sys.stderr.write("streamwright: 'connectors -i' needs at least one connector KEY\n")
+            return 2
+        return catalog.install_many(query, assume_yes=args.yes, hub_url=args.hub_url)
+
+    words = [action, *query]
+    if norm_action in ("find", "search", "s", "-s", "-f", "--find", "--search"):
+        words = query
+    elif set(words) <= set(components.available()):
+        # installed connector names show those connectors, as `pip show NAME` does (`sw connectors google_ads s3`)
+        print("\n".join(_connector_lines(set(words))))
+        return 0
+    return _search(" ".join(words), args.hub_url)  # any other words search the catalog (`sw connectors ads`)
+
+
+def _search(text, hub_url):
+    hits = catalog.search(text, hub_url)
+    if not hits:
+        print("no connectors match %r" % text)
+    for key, entry in sorted(hits.items()):
+        print("  %-16s %-10s %s" % (key, entry.get("trust", "community"), entry.get("summary", "")))
+    return 0

streamwright-0.1.10/src/streamwright/core/cli/commands/jobs.py

@@ -0,0 +1,27 @@
+"""Job-kind commands: a manifest's kind runs its installed JobEngine, and `duckdb` is the DuckDBJob shorthand."""
+
+import argparse
+import sys
+
+from streamwright.core.jobs.engine import load_job_engine
+from streamwright.core.runtime import plugins
+
+
+def register(subparsers):
+    parser = subparsers.add_parser("duckdb", help="run an installed DuckDBJob engine", add_help=False)
+    parser.add_argument("arguments", nargs=argparse.REMAINDER)
+    parser.set_defaults(command_handler=lambda args: execute(args.arguments))
+
+
+def execute(arguments):
+    return run_job_kind("DuckDBJob", arguments)
+
+
+def run_job_kind(kind, argv):
+    """Runs the installed JobEngine of manifest `kind` on the original arguments (no engine is imported by core)."""
+    try:
+        job_engine = load_job_engine(kind)
+    except plugins.PluginError as exc:
+        sys.stderr.write("streamwright: %s\n" % exc)
+        return 2
+    return job_engine.main(argv) or 0