streamwright 0.1.4__tar.gz → 0.1.5__tar.gz

{streamwright-0.1.4/src/streamwright.egg-info → streamwright-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.4
+Version: 0.1.5
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author: Karthick Jaganathan
 License-Expression: Apache-2.0
@@ -57,6 +57,10 @@ streamwright connectors install google_ads   # an API connector (pulls in its SD
 streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
+The catalog comes from the live [StreamWright hub](https://github.com/karthick-jaganathan/streamwright-hub). When it is
+unreachable, the CLI warns and uses the last copy it fetched, else the one bundled with this release;
+`--hub-url bundled` uses the bundled copy only, with no network (reproducible builds).
+
 ## Quickstart
 
 ```bash

{streamwright-0.1.4 → streamwright-0.1.5}/README.md

@@ -29,6 +29,10 @@ streamwright connectors install google_ads   # an API connector (pulls in its SD
 streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
+The catalog comes from the live [StreamWright hub](https://github.com/karthick-jaganathan/streamwright-hub). When it is
+unreachable, the CLI warns and uses the last copy it fetched, else the one bundled with this release;
+`--hub-url bundled` uses the bundled copy only, with no network (reproducible builds).
+
 ## Quickstart
 
 ```bash

{streamwright-0.1.4 → streamwright-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "streamwright"
-version = "0.1.4"
+version = "0.1.5"
 description = "Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere."
 readme = "README.md"
 license = "Apache-2.0"
@@ -58,5 +58,5 @@ packages = [
 ]
 
 [tool.setuptools.package-data]
-# the bundled connector catalog (default for `streamwright connectors`, refreshed from $STREAMWRIGHT_HUB_URL)
+# the bundled connector catalog (the fallback of `streamwright connectors` when the live hub is unreachable)
 "streamwright.core" = ["connectors.json"]

{streamwright-0.1.4 → streamwright-0.1.5}/src/streamwright/core/catalog.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python
 """The connector catalog: discover and install connectors from the StreamWright hub.
 
-A catalog is a static index (bundled with streamwright, optionally refreshed from a
-remote hub URL) that maps a connector key to how it's installed - a PyPI
-requirement, a pinned git URL, or an image. The key is the same name used in
+A catalog is a static index that maps a connector key to how it's installed - a
+PyPI requirement, a pinned git URL, or an image. It is read from the live hub
+(DEFAULT_HUB, or $STREAMWRIGHT_HUB_URL / --hub-url); when the hub is unreachable,
+from the last copy fetched (~/.streamwright/connectors.json), else from the copy
+bundled with this release - with a warning either way. The hub URL "bundled" uses
+the bundled copy only (no network): for reproducible builds. The key is the same name used in
 source YAML (`provider`/`sdk`); the install source is just a delivery detail, so
 a package can move from git to PyPI without changing what users type.
 
@@ -24,7 +27,10 @@ except ImportError:  # pragma: no cover
 from streamwright.core.runtime import components
 
 HUB_ENV = "STREAMWRIGHT_HUB_URL"
+DEFAULT_HUB = "https://karthick-jaganathan.github.io/streamwright-hub/index.json"
+BUNDLED = "bundled"  # a hub URL that means: the catalog bundled with this release, no network
 CACHE = Path(os.path.expanduser("~/.streamwright/connectors.json"))
+TIMEOUT_S = 5
 
 
 def _bundled():
@@ -32,12 +38,18 @@ def _bundled():
     return json.loads(text)
 
 
+def _valid(data):
+    return isinstance(data, dict) and isinstance(data.get("connectors"), dict)
+
+
 def _remote(url):
     import requests
 
-    response = requests.get(url, timeout=10)
+    response = requests.get(url, timeout=TIMEOUT_S)
     response.raise_for_status()
     data = response.json()
+    if not _valid(data):
+        raise ValueError("not a connector catalog (no `connectors` mapping)")
     try:
         CACHE.parent.mkdir(parents=True, exist_ok=True)
         CACHE.write_text(json.dumps(data))
@@ -47,21 +59,41 @@ def _remote(url):
 
 
 def load(hub_url=None):
-    """The catalog: the remote hub if configured (cached), else the bundled default."""
-    url = hub_url or os.environ.get(HUB_ENV)
-    if url:
-        try:
-            return _remote(url)
-        except Exception:  # offline or hub down: fall back
-            pass
+    """
+    The catalog: the live hub (`hub_url`, else $STREAMWRIGHT_HUB_URL, else DEFAULT_HUB; each fetch is cached). When the
+    hub cannot be read, the cached copy, else the bundled one, with a warning on stderr. `bundled`: the bundled copy only.
+    """
+    url = hub_url or os.environ.get(HUB_ENV) or DEFAULT_HUB
+    if url == BUNDLED:
+        return _bundled()
+    try:
+        return _remote(url)
+    except Exception as exc:  # offline, hub down or a bad publish: fall back, visibly
+        reason = type(exc).__name__  # network errors: the type is enough; an HTTP status or a bad index says why
+        status = getattr(getattr(exc, "response", None), "status_code", None)
+        if status:
+            reason = "HTTP %s" % status
+        elif isinstance(exc, ValueError):
+            reason += ": %s" % str(exc)[:100]
     if CACHE.exists():
         try:
-            return json.loads(CACHE.read_text())
+            data = json.loads(CACHE.read_text())
+            if _valid(data):
+                sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the copy fetched on %s\n"
+                                 % (url, reason, _mtime(CACHE)))
+                return data
         except Exception:
             pass
+    sys.stderr.write("streamwright: the connector hub %s is unreachable (%s); using the catalog bundled with this "
+                     "release\n" % (url, reason))
     return _bundled()
 
 
+def _mtime(path):
+    import datetime
+    return datetime.datetime.fromtimestamp(path.stat().st_mtime).strftime("%Y-%m-%d %H:%M")
+
+
 def connectors(hub_url=None):
     return load(hub_url).get("connectors", {})
 

{streamwright-0.1.4 → streamwright-0.1.5}/src/streamwright/core/cli.py

@@ -247,7 +247,9 @@ def _parser():
                       help="connector keys (install: one or more) or words (search)")
     conn.add_argument("--yes", action="store_true", help="skip the confirmation prompt when installing")
     conn.add_argument("--hub-url", metavar="URL",
-                      help="catalog URL to use (default: the bundled catalog, or $STREAMWRIGHT_HUB_URL)")
+                      help="the connector hub (default: $STREAMWRIGHT_HUB_URL, else the live StreamWright hub; when it is "
+                           "unreachable, the last fetched or the bundled catalog, with a warning). 'bundled': the "
+                           "catalog bundled with this release, no network")
 
     validate = commands.add_parser("validate", help="check configuration files: streamwright-validate, plus the checks of "
                                                      "the installed connectors and query builders", add_help=False)

{streamwright-0.1.4 → streamwright-0.1.5/src/streamwright.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamwright
-Version: 0.1.4
+Version: 0.1.5
 Summary: Declarative data connectors — pull from APIs, files and databases with YAML, shape with SQL, write anywhere.
 Author: Karthick Jaganathan
 License-Expression: Apache-2.0
@@ -57,6 +57,10 @@ streamwright connectors install google_ads   # an API connector (pulls in its SD
 streamwright connectors install google_ads meta_ads microsoft_ads   # several at once (one pip install)
 ```
 
+The catalog comes from the live [StreamWright hub](https://github.com/karthick-jaganathan/streamwright-hub). When it is
+unreachable, the CLI warns and uses the last copy it fetched, else the one bundled with this release;
+`--hub-url bundled` uses the bundled copy only, with no network (reproducible builds).
+
 ## Quickstart
 
 ```bash

{streamwright-0.1.4 → streamwright-0.1.5}/tests/test_catalog.py

@@ -1,6 +1,7 @@
 """`streamwright connectors install KEY [KEY ...]` and search: resolution, consent and the single pip command."""
 
 import io
+import json
 import sys
 
 import pytest
@@ -125,3 +126,79 @@ def test_cli_search_joins_its_words(pip, capsys):
     assert cli.main(["connectors", "search", "meta", "insights"]) == 0
     out = capsys.readouterr().out
     assert "meta_ads" in out and "google_ads" not in out
+
+
+# --- where the catalog comes from: the live hub, else the last fetched copy, else the bundled one ---
+
+GOOD = {"schema_version": 1, "connectors": {"files": CATALOG["files"]}}
+
+
+class Response:
+    def __init__(self, data, status=200):
+        self.data, self.status = data, status
+
+    def raise_for_status(self):
+        if self.status >= 400:
+            raise RuntimeError("HTTP %d" % self.status)
+
+    def json(self):
+        return self.data
+
+
+@pytest.fixture
+def hub(monkeypatch, tmp_path):
+    """No $STREAMWRIGHT_HUB_URL, a private cache file, and requests.get answering from `responses` (url -> data)."""
+    monkeypatch.delenv(catalog.HUB_ENV, raising=False)
+    monkeypatch.setattr(catalog, "CACHE", tmp_path / "connectors.json")
+    state = {"responses": {}, "urls": []}
+
+    def get(url, timeout):
+        state["urls"].append(url)
+        if url not in state["responses"]:
+            raise ConnectionError("cannot reach %s" % url)
+        return Response(state["responses"][url])
+    monkeypatch.setattr("requests.get", get)
+    return state
+
+
+def test_the_live_hub_is_the_default_and_is_cached(hub, capsys):
+    hub["responses"][catalog.DEFAULT_HUB] = GOOD
+    assert catalog.load() == GOOD
+    assert hub["urls"] == [catalog.DEFAULT_HUB]
+    assert json.loads(catalog.CACHE.read_text()) == GOOD
+    assert capsys.readouterr().err == ""
+
+
+def test_the_hub_url_can_be_set(hub, monkeypatch):
+    hub["responses"]["https://hub.example/a.json"] = GOOD
+    hub["responses"]["https://hub.example/b.json"] = GOOD
+    monkeypatch.setenv(catalog.HUB_ENV, "https://hub.example/a.json")
+    catalog.load()
+    catalog.load("https://hub.example/b.json")                    # --hub-url wins over the environment
+    assert hub["urls"] == ["https://hub.example/a.json", "https://hub.example/b.json"]
+
+
+def test_bundled_uses_no_network(hub):
+    data = catalog.load(catalog.BUNDLED)
+    assert hub["urls"] == [] and data == catalog._bundled()
+
+
+def test_an_unreachable_hub_falls_back_to_the_last_fetched_copy_with_a_warning(hub, capsys):
+    catalog.CACHE.write_text(json.dumps(GOOD))
+    assert catalog.load() == GOOD
+    err = capsys.readouterr().err
+    assert "the connector hub %s is unreachable (ConnectionError" % catalog.DEFAULT_HUB in err
+    assert "using the copy fetched on" in err
+
+
+def test_with_no_fetched_copy_the_bundled_catalog_is_used_with_a_warning(hub, capsys):
+    assert catalog.load() == catalog._bundled()
+    assert "using the catalog bundled with this release" in capsys.readouterr().err
+
+
+def test_a_malformed_index_is_not_used_nor_cached(hub, capsys):
+    catalog.CACHE.write_text(json.dumps(GOOD))
+    hub["responses"][catalog.DEFAULT_HUB] = {"oops": True}
+    assert catalog.load() == GOOD                                   # the good copy, not the bad publish
+    assert json.loads(catalog.CACHE.read_text()) == GOOD
+    assert "not a connector catalog" in capsys.readouterr().err