streamwright 0.1.3__tar.gz → 0.1.4__tar.gz

{streamwright-0.1.3/src/streamwright.egg-info → streamwright-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.3
+Version: 0.1.4
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author: Karthick Jaganathan
 License-Expression: Apache-2.0
@@ -54,6 +54,7 @@ connectors yet. Add the ones you need on demand:
 streamwright connectors list                 # browse the hub
 streamwright connectors install files        # a reader (files, s3, gcs, postgres)
 streamwright connectors install google_ads   # an API connector (pulls in its SDK)
+streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
 ## Quickstart

{streamwright-0.1.3 → streamwright-0.1.4}/README.md

@@ -26,6 +26,7 @@ connectors yet. Add the ones you need on demand:
 streamwright connectors list                 # browse the hub
 streamwright connectors install files        # a reader (files, s3, gcs, postgres)
 streamwright connectors install google_ads   # an API connector (pulls in its SDK)
+streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
 ## Quickstart

{streamwright-0.1.3 → streamwright-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.3"
+version = "0.1.4"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = "Apache-2.0"

{streamwright-0.1.3 → streamwright-0.1.4}/src/streamwright/core/catalog.py

@@ -74,12 +74,13 @@ def installed():
 
 
 def search(text, hub_url=None):
-    text = (text or "").lower()
+    """The catalog connectors whose key, title, summary or family contains every word of `text`."""
+    words = (text or "").lower().split()
     hits = {}
     for key, entry in connectors(hub_url).items():
         haystack = " ".join([key, entry.get("title", ""), entry.get("summary", ""),
                              entry.get("family", "")]).lower()
-        if text in haystack:
+        if all(word in haystack for word in words):
             hits[key] = entry
     return hits
 
@@ -111,40 +112,62 @@ def listing(hub_url=None):
 
 
 def install(key, assume_yes=False, hub_url=None, pip_args=()):
-    """Resolve `key` in the catalog and install it (pip for pip/git; images are run, not installed)."""
-    entry = connectors(hub_url).get(key)
-    if not entry:
-        sys.stderr.write("streamwright: unknown connector %r (try: streamwright connectors list)\n" % key)
+    """Resolve one connector `key` in the catalog and install it (see install_many)."""
+    return install_many([key], assume_yes=assume_yes, hub_url=hub_url, pip_args=pip_args)
+
+
+def install_many(keys, assume_yes=False, hub_url=None, pip_args=()):
+    """
+    Resolve connector `keys` in the catalog and install them with ONE pip command, so pip resolves their
+    dependencies together. Every key is checked first: an unknown key, or a connector without an installable
+    source, installs nothing. Image-distributed connectors are reported, not installed. Non-"official"
+    connectors need one confirmation for all of them (or `assume_yes`), and are refused non-interactively.
+    """
+    keys = list(dict.fromkeys(keys))  # in order, without repeats
+    if not keys:
+        sys.stderr.write("streamwright: 'connectors install' needs at least one connector KEY\n")
+        return 2
+    catalog = connectors(hub_url)
+    unknown = [key for key in keys if key not in catalog]
+    if unknown:
+        sys.stderr.write("streamwright: unknown connector%s %s (try: streamwright connectors list)\n" % (
+            "s" if len(unknown) > 1 else "", ", ".join(repr(key) for key in unknown)))
         return 2
-    inst = entry.get("install", {})
-    trust = entry.get("trust", "community")
 
-    if "image" in inst:
-        print("%r is distributed as an image, not a Python package:" % key)
-        print("  %s" % inst["image"])
-        print("Run it with the docker or k8s execution mode (it is not pip-installed).")
+    specs, untrusted = [], []
+    for key in keys:
+        entry = catalog[key]
+        inst = entry.get("install", {})
+        if "image" in inst:
+            print("%r is distributed as an image, not a Python package:" % key)
+            print("  %s" % inst["image"])
+            print("Run it with the docker or k8s execution mode (it is not pip-installed).")
+            continue
+        spec = inst.get("pip") or inst.get("git")
+        if not spec:
+            sys.stderr.write("streamwright: connector %r has no installable source\n" % key)
+            return 2
+        specs.append(spec)
+        trust = entry.get("trust", "community")
+        if trust != "official":
+            untrusted.append((key, trust, _source_of(inst)))
+    if not specs:
         return 0
 
-    spec = inst.get("pip") or inst.get("git")
-    if not spec:
-        sys.stderr.write("streamwright: connector %r has no installable source\n" % key)
-        return 2
-
-    if trust != "official" and not assume_yes:
-        source = _source_of(inst)
+    if untrusted and not assume_yes:
+        names = ", ".join("%s connector %r" % (trust, key) for key, trust, _ in untrusted)
         if not sys.stdin.isatty():
-            sys.stderr.write(
-                "streamwright: refusing to install %s connector %r non-interactively; pass --yes to confirm (%s)\n"
-                % (trust, key, source))
+            sys.stderr.write("streamwright: refusing to install %s non-interactively; pass --yes to confirm (%s)\n"
+                             % (names, "; ".join(source for _, _, source in untrusted)))
             return 2
-        sys.stderr.write(
-            "\n  WARNING: %r is a %s connector. Installing runs third-party code from:\n    %s\n"
-            % (key, trust, source))
+        sys.stderr.write("\n  WARNING: installing runs third-party code from:\n")
+        for key, trust, source in untrusted:
+            sys.stderr.write("    %s (%s): %s\n" % (key, trust, source))
         answer = input("  Continue? [y/N] ").strip().lower()
         if answer not in ("y", "yes"):
             print("aborted")
             return 1
 
-    cmd = [sys.executable, "-m", "pip", "install", *pip_args, spec]
-    print("+ " + " ".join(cmd))
+    cmd = [sys.executable, "-m", "pip", "install", *pip_args, *specs]
+    print("+ " + " ".join(cmd), flush=True)
     return subprocess.call(cmd)

{streamwright-0.1.3 → streamwright-0.1.4}/src/streamwright/core/cli.py

@@ -22,6 +22,8 @@ The `streamwright` command.
             [--file-name TEMPLATE] [--allow-connector NAME] [--summary FILE] [logging options]
             (SOURCE: a source folder - source.yaml and streams/ - or a source file)
   streamwright connectors              (installed connectors, with the names of their SDK loggers)
+  streamwright connectors list | search WORDS | install KEY [KEY ...] [--yes] [--hub-url URL]
+                                       (the catalog; install resolves every KEY, then runs one pip install)
   streamwright validate PATH...        (streamwright-validate, plus the checks of the installed connectors and query builders)
 
 Logging options (streamwright run and streamwright validate): --log-level LEVEL, --log NAME=LEVEL, --log-format text|json,
@@ -240,9 +242,9 @@ def _parser():
                                help="list, search or install connectors (bare: the installed connectors and their "
                                     "SDK loggers)")
     conn.add_argument("action", nargs="?", choices=["list", "search", "install"],
-                      help="list (catalog + installed), search TEXT, or install KEY")
-    conn.add_argument("query", nargs="?", metavar="KEY|TEXT",
-                      help="a connector key (install) or text (search)")
+                      help="list (catalog + installed), search TEXT, or install KEY [KEY ...]")
+    conn.add_argument("query", nargs="*", metavar="KEY|TEXT",
+                      help="connector keys (install: one or more) or words (search)")
     conn.add_argument("--yes", action="store_true", help="skip the confirmation prompt when installing")
     conn.add_argument("--hub-url", metavar="URL",
                       help="catalog URL to use (default: the bundled catalog, or $STREAMWRIGHT_HUB_URL)")
@@ -544,18 +546,19 @@ def _connectors_command(args):
         print("\n".join(lines) if lines else "the catalog is empty")
         return 0
     if action == "search":
-        hits = catalog.search(args.query or "", args.hub_url)
+        text = " ".join(args.query)
+        hits = catalog.search(text, args.hub_url)
         if not hits:
-            print("no connectors match %r" % (args.query or ""))
+            print("no connectors match %r" % text)
             return 0
         for key, entry in sorted(hits.items()):
             print("  %-16s %-10s %s" % (key, entry.get("trust", "community"), entry.get("summary", "")))
         return 0
     if action == "install":
         if not args.query:
-            sys.stderr.write("streamwright: 'connectors install' needs a connector KEY\n")
+            sys.stderr.write("streamwright: 'connectors install' needs at least one connector KEY\n")
             return 2
-        return catalog.install(args.query, assume_yes=args.yes, hub_url=args.hub_url)
+        return catalog.install_many(args.query, assume_yes=args.yes, hub_url=args.hub_url)
     return 2
 
 

{streamwright-0.1.3 → streamwright-0.1.4/src/streamwright.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.3
+Version: 0.1.4
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author: Karthick Jaganathan
 License-Expression: Apache-2.0
@@ -54,6 +54,7 @@ connectors yet. Add the ones you need on demand:
 streamwright connectors list                 # browse the hub
 streamwright connectors install files        # a reader (files, s3, gcs, postgres)
 streamwright connectors install google_ads   # an API connector (pulls in its SDK)
+streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
 ## Quickstart

{streamwright-0.1.3 → streamwright-0.1.4}/src/streamwright.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ src/streamwright/core/validation/__init__.py
 src/streamwright/core/validation/engine.py
 src/streamwright/core/validation/schema.py
 src/streamwright/core/validation/source.py
+tests/test_catalog.py
 tests/test_connectors.py
 tests/test_dlt_output.py
 tests/test_exact_decimals.py

streamwright-0.1.4/tests/test_catalog.py

@@ -0,0 +1,127 @@
+"""`streamwright connectors install KEY [KEY ...]` and search: resolution, consent and the single pip command."""
+
+import io
+import sys
+
+import pytest
+
+from streamwright.core import catalog, cli
+
+CATALOG = {
+    "files": {"title": "Files", "family": "readers", "summary": "Local files", "trust": "official",
+              "install": {"git": "git+https://example.test/sw.git@abc#subdirectory=connectors/readers/files"}},
+    "google_ads": {"title": "Google Ads", "family": "ads", "summary": "Google Ads reports", "trust": "official",
+                   "install": {"git": "git+https://example.test/sw.git@abc#subdirectory=connectors/ads/google_ads"}},
+    "meta_ads": {"title": "Meta Ads", "family": "ads", "summary": "Meta Marketing insights", "trust": "official",
+                 "install": {"pip": "streamwright-meta-ads>=0.1,<0.2"}},
+    "acme": {"title": "Acme", "family": "ads", "summary": "Acme reports", "trust": "community",
+             "install": {"git": "git+https://example.test/acme.git@def"}},
+    "imaged": {"title": "Imaged", "family": "readers", "summary": "An image", "trust": "official",
+               "install": {"image": "ghcr.io/org/imaged@sha256:" + "0" * 64}},
+    "empty": {"title": "Empty", "family": "readers", "summary": "No source", "trust": "official", "install": {}},
+}
+
+
+class TTY(io.StringIO):
+    def __init__(self, tty):
+        super().__init__()
+        self.tty = tty
+
+    def isatty(self):
+        return self.tty
+
+
+@pytest.fixture
+def pip(monkeypatch):
+    """The catalog above; records each pip command instead of running it."""
+    calls = []
+    monkeypatch.setattr(catalog, "connectors", lambda hub_url=None: CATALOG)
+    monkeypatch.setattr(catalog.subprocess, "call", lambda cmd: calls.append(cmd) or 0)
+    monkeypatch.setattr(sys, "stdin", TTY(False))
+    return calls
+
+
+def specs(cmd):
+    return cmd[cmd.index("install") + 1:]
+
+
+def test_one_key_installs_its_source(pip):
+    assert catalog.install("files") == 0
+    assert [specs(cmd) for cmd in pip] == [[CATALOG["files"]["install"]["git"]]]
+    assert pip[0][:4] == [sys.executable, "-m", "pip", "install"]
+
+
+def test_several_keys_install_with_one_pip_command_in_order_without_repeats(pip):
+    assert catalog.install_many(["google_ads", "meta_ads", "files", "google_ads"]) == 0
+    assert len(pip) == 1
+    assert specs(pip[0]) == [CATALOG["google_ads"]["install"]["git"], "streamwright-meta-ads>=0.1,<0.2",
+                             CATALOG["files"]["install"]["git"]]
+
+
+def test_an_unknown_key_installs_nothing(pip, capsys):
+    assert catalog.install_many(["google_ads", "nope", "nada"]) == 2
+    assert pip == []
+    assert "unknown connectors 'nope', 'nada'" in capsys.readouterr().err
+
+
+def test_a_connector_without_a_source_installs_nothing(pip, capsys):
+    assert catalog.install_many(["files", "empty"]) == 2
+    assert pip == []
+    assert "'empty' has no installable source" in capsys.readouterr().err
+
+
+def test_no_keys_is_an_error(pip, capsys):
+    assert catalog.install_many([]) == 2
+    assert pip == []
+    assert "at least one connector KEY" in capsys.readouterr().err
+
+
+def test_a_community_connector_is_refused_non_interactively_without_yes(pip, capsys):
+    assert catalog.install_many(["files", "acme"]) == 2
+    assert pip == []
+    assert "refusing to install community connector 'acme' non-interactively" in capsys.readouterr().err
+    assert catalog.install_many(["files", "acme"], assume_yes=True) == 0
+    assert specs(pip[0]) == [CATALOG["files"]["install"]["git"], CATALOG["acme"]["install"]["git"]]
+
+
+def test_one_confirmation_covers_every_untrusted_connector(pip, monkeypatch, capsys):
+    monkeypatch.setattr(sys, "stdin", TTY(True))
+    answers = []
+    monkeypatch.setattr("builtins.input", lambda prompt: answers.append(prompt) or "n")
+    assert catalog.install_many(["acme", "files"]) == 1
+    assert pip == [] and len(answers) == 1
+    monkeypatch.setattr("builtins.input", lambda prompt: "y")
+    assert catalog.install_many(["acme", "files"]) == 0
+    assert len(pip) == 1
+    assert "acme (community): git: git+https://example.test/acme.git@def" in capsys.readouterr().err
+
+
+def test_images_are_reported_not_installed(pip, capsys):
+    assert catalog.install_many(["imaged"]) == 0
+    assert pip == []
+    assert catalog.install_many(["imaged", "files"]) == 0
+    assert specs(pip[0]) == [CATALOG["files"]["install"]["git"]]
+    assert "'imaged' is distributed as an image" in capsys.readouterr().out
+
+
+def test_search_matches_every_word(pip):
+    assert set(catalog.search("ads")) == {"google_ads", "meta_ads", "acme"}
+    assert set(catalog.search("google ads")) == {"google_ads"}
+    assert set(catalog.search("")) == set(CATALOG)
+
+
+def test_cli_installs_several_connectors(pip):
+    assert cli.main(["connectors", "install", "google_ads", "meta_ads", "--yes"]) == 0
+    assert specs(pip[0]) == [CATALOG["google_ads"]["install"]["git"], "streamwright-meta-ads>=0.1,<0.2"]
+
+
+def test_cli_install_needs_a_key(pip, capsys):
+    assert cli.main(["connectors", "install"]) == 2
+    assert pip == []
+    assert "at least one connector KEY" in capsys.readouterr().err
+
+
+def test_cli_search_joins_its_words(pip, capsys):
+    assert cli.main(["connectors", "search", "meta", "insights"]) == 0
+    out = capsys.readouterr().out
+    assert "meta_ads" in out and "google_ads" not in out