streamwright 0.1.3__tar.gz → 0.1.5__tar.gz

{streamwright-0.1.3/src/streamwright.egg-info → streamwright-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.3
+Version: 0.1.5
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author: Karthick Jaganathan
 License-Expression: Apache-2.0
@@ -54,8 +54,13 @@ connectors yet. Add the ones you need on demand:
 streamwright connectors list                 # browse the hub
 streamwright connectors install files        # a reader (files, s3, gcs, postgres)
 streamwright connectors install google_ads   # an API connector (pulls in its SDK)
+streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
+The catalog comes from the live [StreamWright hub](https://github.com/karthick-jaganathan/streamwright-hub). When it is
+unreachable, the CLI warns and uses the last copy it fetched, else the one bundled with this release;
+`--hub-url bundled` uses the bundled copy only, with no network (reproducible builds).
+
 ## Quickstart
 
 ```bash

{streamwright-0.1.3 → streamwright-0.1.5}/README.md

@@ -26,8 +26,13 @@ connectors yet. Add the ones you need on demand:
 streamwright connectors list                 # browse the hub
 streamwright connectors install files        # a reader (files, s3, gcs, postgres)
 streamwright connectors install google_ads   # an API connector (pulls in its SDK)
+streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
+The catalog comes from the live [StreamWright hub](https://github.com/karthick-jaganathan/streamwright-hub). When it is
+unreachable, the CLI warns and uses the last copy it fetched, else the one bundled with this release;
+`--hub-url bundled` uses the bundled copy only, with no network (reproducible builds).
+
 ## Quickstart
 
 ```bash

{streamwright-0.1.3 → streamwright-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.3"
+version = "0.1.5"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = "Apache-2.0"
@@ -58,5 +58,5 @@ packages = [
 ]
 
 [tool.setuptools.package-data]
-# the bundled connector catalog (default for `streamwright connectors`, refreshed from $STREAMWRIGHT_HUB_URL)
+# the bundled connector catalog (the fallback of `streamwright connectors` when the live hub is unreachable)
 "streamwright.core" = ["connectors.json"]

streamwright-0.1.5/src/streamwright/core/catalog.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+"""The connector catalog: discover and install connectors from the StreamWright hub.
+
+A catalog is a static index that maps a connector key to how it's installed - a
+PyPI requirement, a pinned git URL, or an image. It is read from the live hub
+(DEFAULT_HUB, or $STREAMWRIGHT_HUB_URL / --hub-url); when the hub is unreachable,
+from the last copy fetched (~/.streamwright/connectors.json), else from the copy
+bundled with this release - with a warning either way. The hub URL "bundled" uses
+the bundled copy only (no network): for reproducible builds. The key is the same name used in
+source YAML (`provider`/`sdk`); the install source is just a delivery detail, so
+a package can move from git to PyPI without changing what users type.
+
+Security: non-"official" connectors run third-party code, so `install` asks for
+confirmation (or `--yes`) and refuses non-interactive installs without it.
+"""
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+try:
+    from importlib import resources
+except ImportError:  # pragma: no cover
+    import importlib_resources as resources
+
+from streamwright.core.runtime import components
+
+HUB_ENV = "STREAMWRIGHT_HUB_URL"
+DEFAULT_HUB = "https://karthick-jaganathan.github.io/streamwright-hub/index.json"
+BUNDLED = "bundled"  # a hub URL that means: the catalog bundled with this release, no network
+CACHE = Path(os.path.expanduser("~/.streamwright/connectors.json"))
+TIMEOUT_S = 5
+
+
+def _bundled():
+    text = resources.files("streamwright.core").joinpath("connectors.json").read_text()
+    return json.loads(text)
+
+
+def _valid(data):
+    return isinstance(data, dict) and isinstance(data.get("connectors"), dict)
+
+
+def _remote(url):
+    import requests
+
+    response = requests.get(url, timeout=TIMEOUT_S)
+    response.raise_for_status()
+    data = response.json()
+    if not _valid(data):
+        raise ValueError("not a connector catalog (no `connectors` mapping)")
+    try:
+        CACHE.parent.mkdir(parents=True, exist_ok=True)
+        CACHE.write_text(json.dumps(data))
+    except OSError:
+        pass
+    return data
+
+
+def load(hub_url=None):
+    """
+    The catalog: the live hub (`hub_url`, else $STREAMWRIGHT_HUB_URL, else DEFAULT_HUB; each fetch is cached). When the
+    hub cannot be read, the cached copy, else the bundled one, with a warning on stderr. `bundled`: the bundled copy only.
+    """
+    url = hub_url or os.environ.get(HUB_ENV) or DEFAULT_HUB
+    if url == BUNDLED:
+        return _bundled()
+    try:
+        return _remote(url)
+    except Exception as exc:  # offline, hub down or a bad publish: fall back, visibly
+        reason = type(exc).__name__  # network errors: the type is enough; an HTTP status or a bad index says why
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+        if status:
+            reason = "HTTP %s" % status
+        elif isinstance(exc, ValueError):
+            reason += ": %s" % str(exc)[:100]
+    if CACHE.exists():
+        try:
+            data = json.loads(CACHE.read_text())
+            if _valid(data):
+                sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the copy fetched on %s\n"
+                                 % (url, reason, _mtime(CACHE)))
+                return data
+        except Exception:
+            pass
+    sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the catalog bundled with this "
+                     "release\n" % (url, reason))
+    return _bundled()
+
+
+def _mtime(path):
+    import datetime
+    return datetime.datetime.fromtimestamp(path.stat().st_mtime).strftime("%Y-%m-%d %H:%M")
+
+
+def connectors(hub_url=None):
+    return load(hub_url).get("connectors", {})
+
+
+def installed():
+    try:
+        return set(components.available())
+    except Exception:
+        return set()
+
+
+def search(text, hub_url=None):
+    """The catalog connectors whose key, title, summary or family contains every word of `text`."""
+    words = (text or "").lower().split()
+    hits = {}
+    for key, entry in connectors(hub_url).items():
+        haystack = " ".join([key, entry.get("title", ""), entry.get("summary", ""),
+                             entry.get("family", "")]).lower()
+        if all(word in haystack for word in words):
+            hits[key] = entry
+    return hits
+
+
+def _source_of(install):
+    if "pip" in install:
+        return "PyPI: " + install["pip"]
+    if "git" in install:
+        return "git: " + install["git"]
+    if "image" in install:
+        return "image: " + install["image"]
+    return "unknown"
+
+
+def listing(hub_url=None):
+    """Lines for `streamwright connectors list`: every catalog connector with install state + trust."""
+    have = installed()
+    lines = []
+    by_family = {}
+    for key, entry in connectors(hub_url).items():
+        by_family.setdefault(entry.get("family", "other"), []).append((key, entry))
+    for family in sorted(by_family):
+        lines.append("%s:" % family)
+        for key, entry in sorted(by_family[family]):
+            mark = "[installed]" if key in have else "          "
+            trust = entry.get("trust", "community")
+            lines.append("  %s %-16s %-10s %s" % (mark, key, trust, entry.get("summary", "")))
+    return lines
+
+
+def install(key, assume_yes=False, hub_url=None, pip_args=()):
+    """Resolve one connector `key` in the catalog and install it (see install_many)."""
+    return install_many([key], assume_yes=assume_yes, hub_url=hub_url, pip_args=pip_args)
+
+
+def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
+    """
+    Resolve connector `keys` in the catalog and install them with ONE pip command, so pip resolves their
+    dependencies together. Every key is checked first: an unknown key, or a connector without an installable
+    source, installs nothing. Image-distributed connectors are reported, not installed. Non-"official"
+    connectors need one confirmation for all of them (or `assume_yes`), and are refused non-interactively.
+    """
+    keys = list(dict.fromkeys(keys))  # in order, without repeats
+    if not keys:
+        sys.stderr.write("streamwright: 'connectors install' needs at least one connector KEY\n")
+        return 2
+    catalog = connectors(hub_url)
+    unknown = [key for key in keys if key not in catalog]
+    if unknown:
+        sys.stderr.write("streamwright: unknown connector%s %s (try: streamwright connectors list)\n" % (
+            "s" if len(unknown) > 1 else "", ", ".join(repr(key) for key in unknown)))
+        return 2
+
+    specs, untrusted = [], []
+    for key in keys:
+        entry = catalog[key]
+        inst = entry.get("install", {})
+        if "image" in inst:
+            print("%r is distributed as an image, not a Python package:" % key)
+            print("  %s" % inst["image"])
+            print("Run it with the docker or k8s execution mode (it is not pip-installed).")
+            continue
+        spec = inst.get("pip") or inst.get("git")
+        if not spec:
+            sys.stderr.write("streamwright: connector %r has no installable source\n" % key)
+            return 2
+        specs.append(spec)
+        trust = entry.get("trust", "community")
+        if trust != "official":
+            untrusted.append((key, trust, _source_of(inst)))
+    if not specs:
+        return 0
+
+    if untrusted and not assume_yes:
+        names = ", ".join("%s connector %r" % (trust, key) for key, trust, _ in untrusted)
+        if not sys.stdin.isatty():
+            sys.stderr.write("streamwright: refusing to install %s non-interactively; pass --yes to confirm (%s)\n"
+                             % (names, "; ".join(source for _, _, source in untrusted)))
+            return 2
+        sys.stderr.write("\n  WARNING: installing runs third-party code from:\n")
+        for key, trust, source in untrusted:
+            sys.stderr.write("    %s (%s): %s\n" % (key, trust, source))
+        answer = input("  Continue? [y/N] ").strip().lower()
+        if answer not in ("y", "yes"):
+            print("aborted")
+            return 1
+
+    cmd = [sys.executable, "-m", "pip", "install", *pip_args, *specs]
+    print("+ " + " ".join(cmd), flush=True)
+    return subprocess.call(cmd)

{streamwright-0.1.3 → streamwright-0.1.5}/src/streamwright/core/cli.py

@@ -22,6 +22,8 @@ The `streamwright` command.
             [--file-name TEMPLATE] [--allow-connector NAME] [--summary FILE] [logging options]
             (SOURCE: a source folder - source.yaml and streams/ - or a source file)
   streamwright connectors              (installed connectors, with the names of their SDK loggers)
+  streamwright connectors list | search WORDS | install KEY [KEY ...] [--yes] [--hub-url URL]
+                                       (the catalog; install resolves every KEY, then runs one pip install)
   streamwright validate PATH...        (streamwright-validate, plus the checks of the installed connectors and query builders)
 
 Logging options (streamwright run and streamwright validate): --log-level LEVEL, --log NAME=LEVEL, --log-format text|json,
@@ -240,12 +242,14 @@ def _parser():
                                help="list, search or install connectors (bare: the installed connectors and their "
                                     "SDK loggers)")
     conn.add_argument("action", nargs="?", choices=["list", "search", "install"],
-                      help="list (catalog + installed), search TEXT, or install KEY")
-    conn.add_argument("query", nargs="?", metavar="KEY|TEXT",
-                      help="a connector key (install) or text (search)")
+                      help="list (catalog + installed), search TEXT, or install KEY [KEY ...]")
+    conn.add_argument("query", nargs="*", metavar="KEY|TEXT",
+                      help="connector keys (install: one or more) or words (search)")
     conn.add_argument("--yes", action="store_true", help="skip the confirmation prompt when installing")
     conn.add_argument("--hub-url", metavar="URL",
-                      help="catalog URL to use (default: the bundled catalog, or $STREAMWRIGHT_HUB_URL)")
+                      help="the connector hub (default: $STREAMWRIGHT_HUB_URL, else the live StreamWright hub; when it is "
+                           "unreachable, the last fetched or the bundled catalog, with a warning). 'bundled': the "
+                           "catalog bundled with this release, no network")
 
     validate = commands.add_parser("validate", help="check configuration files: streamwright-validate, plus the checks of "
                                                      "the installed connectors and query builders", add_help=False)
@@ -544,18 +548,19 @@ def _connectors_command(args):
         print("\n".join(lines) if lines else "the catalog is empty")
         return 0
     if action == "search":
-        hits = catalog.search(args.query or "", args.hub_url)
+        text = " ".join(args.query)
+        hits = catalog.search(text, args.hub_url)
         if not hits:
-            print("no connectors match %r" % (args.query or ""))
+            print("no connectors match %r" % text)
             return 0
         for key, entry in sorted(hits.items()):
             print("  %-16s %-10s %s" % (key, entry.get("trust", "community"), entry.get("summary", "")))
         return 0
     if action == "install":
         if not args.query:
-            sys.stderr.write("streamwright: 'connectors install' needs a connector KEY\n")
+            sys.stderr.write("streamwright: 'connectors install' needs at least one connector KEY\n")
             return 2
-        return catalog.install(args.query, assume_yes=args.yes, hub_url=args.hub_url)
+        return catalog.install_many(args.query, assume_yes=args.yes, hub_url=args.hub_url)
     return 2
 
 

{streamwright-0.1.3 → streamwright-0.1.5/src/streamwright.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.3
+Version: 0.1.5
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author: Karthick Jaganathan
 License-Expression: Apache-2.0
@@ -54,8 +54,13 @@ connectors yet. Add the ones you need on demand:
 streamwright connectors list                 # browse the hub
 streamwright connectors install files        # a reader (files, s3, gcs, postgres)
 streamwright connectors install google_ads   # an API connector (pulls in its SDK)
+streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
+The catalog comes from the live [StreamWright hub](https://github.com/karthick-jaganathan/streamwright-hub). When it is
+unreachable, the CLI warns and uses the last copy it fetched, else the one bundled with this release;
+`--hub-url bundled` uses the bundled copy only, with no network (reproducible builds).
+
 ## Quickstart
 
 ```bash

{streamwright-0.1.3 → streamwright-0.1.5}/src/streamwright.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ src/streamwright/core/validation/__init__.py
 src/streamwright/core/validation/engine.py
 src/streamwright/core/validation/schema.py
 src/streamwright/core/validation/source.py
+tests/test_catalog.py
 tests/test_connectors.py
 tests/test_dlt_output.py
 tests/test_exact_decimals.py

streamwright-0.1.5/tests/test_catalog.py

@@ -0,0 +1,204 @@
+"""`streamwright connectors install KEY [KEY ...]` and search: resolution, consent and the single pip command."""
+
+import io
+import json
+import sys
+
+import pytest
+
+from streamwright.core import catalog, cli
+
+CATALOG = {
+    "files": {"title": "Files", "family": "readers", "summary": "Local files", "trust": "official",
+              "install": {"git": "git+https://example.test/sw.git@abc#subdirectory=connectors/readers/files"}},
+    "google_ads": {"title": "Google Ads", "family": "ads", "summary": "Google Ads reports", "trust": "official",
+                   "install": {"git": "git+https://example.test/sw.git@abc#subdirectory=connectors/ads/google_ads"}},
+    "meta_ads": {"title": "Meta Ads", "family": "ads", "summary": "Meta Marketing insights", "trust": "official",
+                 "install": {"pip": "streamwright-meta-ads>=0.1,<0.2"}},
+    "acme": {"title": "Acme", "family": "ads", "summary": "Acme reports", "trust": "community",
+             "install": {"git": "git+https://example.test/acme.git@def"}},
+    "imaged": {"title": "Imaged", "family": "readers", "summary": "An image", "trust": "official",
+               "install": {"image": "ghcr.io/org/imaged@sha256:" + "0" * 64}},
+    "empty": {"title": "Empty", "family": "readers", "summary": "No source", "trust": "official", "install": {}},
+}
+
+
+class TTY(io.StringIO):
+    def __init__(self, tty):
+        super().__init__()
+        self.tty = tty
+
+    def isatty(self):
+        return self.tty
+
+
+@pytest.fixture
+def pip(monkeypatch):
+    """The catalog above; records each pip command instead of running it."""
+    calls = []
+    monkeypatch.setattr(catalog, "connectors", lambda hub_url=None: CATALOG)
+    monkeypatch.setattr(catalog.subprocess, "call", lambda cmd: calls.append(cmd) or 0)
+    monkeypatch.setattr(sys, "stdin", TTY(False))
+    return calls
+
+
+def specs(cmd):
+    return cmd[cmd.index("install") + 1:]
+
+
+def test_one_key_installs_its_source(pip):
+    assert catalog.install("files") == 0
+    assert [specs(cmd) for cmd in pip] == [[CATALOG["files"]["install"]["git"]]]
+    assert pip[0][:4] == [sys.executable, "-m", "pip", "install"]
+
+
+def test_several_keys_install_with_one_pip_command_in_order_without_repeats(pip):
+    assert catalog.install_many(["google_ads", "meta_ads", "files", "google_ads"]) == 0
+    assert len(pip) == 1
+    assert specs(pip[0]) == [CATALOG["google_ads"]["install"]["git"], "streamwright-meta-ads>=0.1,<0.2",
+                             CATALOG["files"]["install"]["git"]]
+
+
+def test_an_unknown_key_installs_nothing(pip, capsys):
+    assert catalog.install_many(["google_ads", "nope", "nada"]) == 2
+    assert pip == []
+    assert "unknown connectors 'nope', 'nada'" in capsys.readouterr().err
+
+
+def test_a_connector_without_a_source_installs_nothing(pip, capsys):
+    assert catalog.install_many(["files", "empty"]) == 2
+    assert pip == []
+    assert "'empty' has no installable source" in capsys.readouterr().err
+
+
+def test_no_keys_is_an_error(pip, capsys):
+    assert catalog.install_many([]) == 2
+    assert pip == []
+    assert "at least one connector KEY" in capsys.readouterr().err
+
+
+def test_a_community_connector_is_refused_non_interactively_without_yes(pip, capsys):
+    assert catalog.install_many(["files", "acme"]) == 2
+    assert pip == []
+    assert "refusing to install community connector 'acme' non-interactively" in capsys.readouterr().err
+    assert catalog.install_many(["files", "acme"], assume_yes=True) == 0
+    assert specs(pip[0]) == [CATALOG["files"]["install"]["git"], CATALOG["acme"]["install"]["git"]]
+
+
+def test_one_confirmation_covers_every_untrusted_connector(pip, monkeypatch, capsys):
+    monkeypatch.setattr(sys, "stdin", TTY(True))
+    answers = []
+    monkeypatch.setattr("builtins.input", lambda prompt: answers.append(prompt) or "n")
+    assert catalog.install_many(["acme", "files"]) == 1
+    assert pip == [] and len(answers) == 1
+    monkeypatch.setattr("builtins.input", lambda prompt: "y")
+    assert catalog.install_many(["acme", "files"]) == 0
+    assert len(pip) == 1
+    assert "acme (community): git: git+https://example.test/acme.git@def" in capsys.readouterr().err
+
+
+def test_images_are_reported_not_installed(pip, capsys):
+    assert catalog.install_many(["imaged"]) == 0
+    assert pip == []
+    assert catalog.install_many(["imaged", "files"]) == 0
+    assert specs(pip[0]) == [CATALOG["files"]["install"]["git"]]
+    assert "'imaged' is distributed as an image" in capsys.readouterr().out
+
+
+def test_search_matches_every_word(pip):
+    assert set(catalog.search("ads")) == {"google_ads", "meta_ads", "acme"}
+    assert set(catalog.search("google ads")) == {"google_ads"}
+    assert set(catalog.search("")) == set(CATALOG)
+
+
+def test_cli_installs_several_connectors(pip):
+    assert cli.main(["connectors", "install", "google_ads", "meta_ads", "--yes"]) == 0
+    assert specs(pip[0]) == [CATALOG["google_ads"]["install"]["git"], "streamwright-meta-ads>=0.1,<0.2"]
+
+
+def test_cli_install_needs_a_key(pip, capsys):
+    assert cli.main(["connectors", "install"]) == 2
+    assert pip == []
+    assert "at least one connector KEY" in capsys.readouterr().err
+
+
+def test_cli_search_joins_its_words(pip, capsys):
+    assert cli.main(["connectors", "search", "meta", "insights"]) == 0
+    out = capsys.readouterr().out
+    assert "meta_ads" in out and "google_ads" not in out
+
+
+# --- where the catalog comes from: the live hub, else the last fetched copy, else the bundled one ---
+
+GOOD = {"schema_version": 1, "connectors": {"files": CATALOG["files"]}}
+
+
+class Response:
+    def __init__(self, data, status=200):
+        self.data, self.status = data, status
+
+    def raise_for_status(self):
+        if self.status >= 400:
+            raise RuntimeError("HTTP %d" % self.status)
+
+    def json(self):
+        return self.data
+
+
+@pytest.fixture
+def hub(monkeypatch, tmp_path):
+    """No $STREAMWRIGHT_HUB_URL, a private cache file, and requests.get answering from `responses` (url -> data)."""
+    monkeypatch.delenv(catalog.HUB_ENV, raising=False)
+    monkeypatch.setattr(catalog, "CACHE", tmp_path / "connectors.json")
+    state = {"responses": {}, "urls": []}
+
+    def get(url, timeout):
+        state["urls"].append(url)
+        if url not in state["responses"]:
+            raise ConnectionError("cannot reach %s" % url)
+        return Response(state["responses"][url])
+    monkeypatch.setattr("requests.get", get)
+    return state
+
+
+def test_the_live_hub_is_the_default_and_is_cached(hub, capsys):
+    hub["responses"][catalog.DEFAULT_HUB] = GOOD
+    assert catalog.load() == GOOD
+    assert hub["urls"] == [catalog.DEFAULT_HUB]
+    assert json.loads(catalog.CACHE.read_text()) == GOOD
+    assert capsys.readouterr().err == ""
+
+
+def test_the_hub_url_can_be_set(hub, monkeypatch):
+    hub["responses"]["https://hub.example/a.json"] = GOOD
+    hub["responses"]["https://hub.example/b.json"] = GOOD
+    monkeypatch.setenv(catalog.HUB_ENV, "https://hub.example/a.json")
+    catalog.load()
+    catalog.load("https://hub.example/b.json")                    # --hub-url wins over the environment
+    assert hub["urls"] == ["https://hub.example/a.json", "https://hub.example/b.json"]
+
+
+def test_bundled_uses_no_network(hub):
+    data = catalog.load(catalog.BUNDLED)
+    assert hub["urls"] == [] and data == catalog._bundled()
+
+
+def test_an_unreachable_hub_falls_back_to_the_last_fetched_copy_with_a_warning(hub, capsys):
+    catalog.CACHE.write_text(json.dumps(GOOD))
+    assert catalog.load() == GOOD
+    err = capsys.readouterr().err
+    assert "the connector hub %s is unreachable (ConnectionError" % catalog.DEFAULT_HUB in err
+    assert "using the copy fetched on" in err
+
+
+def test_with_no_fetched_copy_the_bundled_catalog_is_used_with_a_warning(hub, capsys):
+    assert catalog.load() == catalog._bundled()
+    assert "using the catalog bundled with this release" in capsys.readouterr().err
+
+
+def test_a_malformed_index_is_not_used_nor_cached(hub, capsys):
+    catalog.CACHE.write_text(json.dumps(GOOD))
+    hub["responses"][catalog.DEFAULT_HUB] = {"oops": True}
+    assert catalog.load() == GOOD                                   # the good copy, not the bad publish
+    assert json.loads(catalog.CACHE.read_text()) == GOOD
+    assert "not a connector catalog" in capsys.readouterr().err

streamwright-0.1.3/src/streamwright/core/catalog.py

@@ -1,150 +0,0 @@
-#!/usr/bin/env python
-"""The connector catalog: discover and install connectors from the StreamWright hub.
-
-A catalog is a static index (bundled with streamwright, optionally refreshed from a
-remote hub URL) that maps a connector key to how it's installed - a PyPI
-requirement, a pinned git URL, or an image. The key is the same name used in
-source YAML (`provider`/`sdk`); the install source is just a delivery detail, so
-a package can move from git to PyPI without changing what users type.
-
-Security: non-"official" connectors run third-party code, so `install` asks for
-confirmation (or `--yes`) and refuses non-interactive installs without it.
-"""
-import json
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-try:
-    from importlib import resources
-except ImportError:  # pragma: no cover
-    import importlib_resources as resources
-
-from streamwright.core.runtime import components
-
-HUB_ENV = "STREAMWRIGHT_HUB_URL"
-CACHE = Path(os.path.expanduser("~/.streamwright/connectors.json"))
-
-
-def _bundled():
-    text = resources.files("streamwright.core").joinpath("connectors.json").read_text()
-    return json.loads(text)
-
-
-def _remote(url):
-    import requests
-
-    response = requests.get(url, timeout=10)
-    response.raise_for_status()
-    data = response.json()
-    try:
-        CACHE.parent.mkdir(parents=True, exist_ok=True)
-        CACHE.write_text(json.dumps(data))
-    except OSError:
-        pass
-    return data
-
-
-def load(hub_url=None):
-    """The catalog: the remote hub if configured (cached), else the bundled default."""
-    url = hub_url or os.environ.get(HUB_ENV)
-    if url:
-        try:
-            return _remote(url)
-        except Exception:  # offline or hub down: fall back
-            pass
-    if CACHE.exists():
-        try:
-            return json.loads(CACHE.read_text())
-        except Exception:
-            pass
-    return _bundled()
-
-
-def connectors(hub_url=None):
-    return load(hub_url).get("connectors", {})
-
-
-def installed():
-    try:
-        return set(components.available())
-    except Exception:
-        return set()
-
-
-def search(text, hub_url=None):
-    text = (text or "").lower()
-    hits = {}
-    for key, entry in connectors(hub_url).items():
-        haystack = " ".join([key, entry.get("title", ""), entry.get("summary", ""),
-                             entry.get("family", "")]).lower()
-        if text in haystack:
-            hits[key] = entry
-    return hits
-
-
-def _source_of(install):
-    if "pip" in install:
-        return "PyPI: " + install["pip"]
-    if "git" in install:
-        return "git: " + install["git"]
-    if "image" in install:
-        return "image: " + install["image"]
-    return "unknown"
-
-
-def listing(hub_url=None):
-    """Lines for `streamwright connectors list`: every catalog connector with install state + trust."""
-    have = installed()
-    lines = []
-    by_family = {}
-    for key, entry in connectors(hub_url).items():
-        by_family.setdefault(entry.get("family", "other"), []).append((key, entry))
-    for family in sorted(by_family):
-        lines.append("%s:" % family)
-        for key, entry in sorted(by_family[family]):
-            mark = "[installed]" if key in have else "          "
-            trust = entry.get("trust", "community")
-            lines.append("  %s %-16s %-10s %s" % (mark, key, trust, entry.get("summary", "")))
-    return lines
-
-
-def install(key, assume_yes=False, hub_url=None, pip_args=()):
-    """Resolve `key` in the catalog and install it (pip for pip/git; images are run, not installed)."""
-    entry = connectors(hub_url).get(key)
-    if not entry:
-        sys.stderr.write("streamwright: unknown connector %r (try: streamwright connectors list)\n" % key)
-        return 2
-    inst = entry.get("install", {})
-    trust = entry.get("trust", "community")
-
-    if "image" in inst:
-        print("%r is distributed as an image, not a Python package:" % key)
-        print("  %s" % inst["image"])
-        print("Run it with the docker or k8s execution mode (it is not pip-installed).")
-        return 0
-
-    spec = inst.get("pip") or inst.get("git")
-    if not spec:
-        sys.stderr.write("streamwright: connector %r has no installable source\n" % key)
-        return 2
-
-    if trust != "official" and not assume_yes:
-        source = _source_of(inst)
-        if not sys.stdin.isatty():
-            sys.stderr.write(
-                "streamwright: refusing to install %s connector %r non-interactively; pass --yes to confirm (%s)\n"
-                % (trust, key, source))
-            return 2
-        sys.stderr.write(
-            "\n  WARNING: %r is a %s connector. Installing runs third-party code from:\n    %s\n"
-            % (key, trust, source))
-        answer = input("  Continue? [y/N] ").strip().lower()
-        if answer not in ("y", "yes"):
-            print("aborted")
-            return 1
-
-    cmd = [sys.executable, "-m", "pip", "install", *pip_args, spec]
-    print("+ " + " ".join(cmd))
-    return subprocess.call(cmd)