specstream-haystack 0.1.0__tar.gz → 0.1.2__tar.gz

{specstream_haystack-0.1.0 → specstream_haystack-0.1.2}/PKG-INFO

@@ -1,16 +1,16 @@
 Metadata-Version: 2.5
 Name: specstream-haystack
-Version: 0.1.0
-Summary: Give a Haystack pipeline or agent any list from the live web: ask in plain words, get every matching row as a Document with the page it came from, within a price you set in code.
+Version: 0.1.2
+Summary: Give a Haystack pipeline or agent lead lists, company lists and grants from the live web. Every row comes with its source, priced per match in code.
 Project-URL: Homepage, https://specstream.ai
 Project-URL: Documentation, https://specstream.ai/docs
-Project-URL: Repository, https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack
-Project-URL: Issues, https://github.com/Specstreamai/specstream-tools/issues
+Project-URL: Repository, https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack
+Project-URL: Issues, https://github.com/specstream/specstream-tools/issues
 Project-URL: Pricing, https://specstream.ai/pricing
 Author-email: SpecStream <support@specstream.ai>
 License-Expression: MIT
 License-File: LICENSE
-Keywords: agents,haystack,haystack-ai,lead-lists,live-web,research,specstream,tools,web-data
+Keywords: agents,company-list,grants,haystack,haystack-ai,lead-generation,lead-list,lead-lists,live-web,research,specstream,tools,web-data
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
@@ -80,4 +80,4 @@ The agent writes the request and picks how many rows. You decide what one list m
 
 The same request with the same limits, in the same hour, reaches the run it already started instead of starting another. A retried step never pays for the list twice.
 
-MIT licensed. Source: [Specstreamai/specstream-tools](https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack).
+MIT licensed. Source: [specstream/specstream-tools](https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack).

{specstream_haystack-0.1.0 → specstream_haystack-0.1.2}/README.md

@@ -47,4 +47,4 @@ The agent writes the request and picks how many rows. You decide what one list m
 
 The same request with the same limits, in the same hour, reaches the run it already started instead of starting another. A retried step never pays for the list twice.
 
-MIT licensed. Source: [Specstreamai/specstream-tools](https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack).
+MIT licensed. Source: [specstream/specstream-tools](https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack).

{specstream_haystack-0.1.0 → specstream_haystack-0.1.2}/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "specstream-haystack"
-version = "0.1.0"
-description = "Give a Haystack pipeline or agent any list from the live web: ask in plain words, get every matching row as a Document with the page it came from, within a price you set in code."
+version = "0.1.2"
+description = "Give a Haystack pipeline or agent lead lists, company lists and grants from the live web. Every row comes with its source, priced per match in code."
 readme = "README.md"
 requires-python = ">=3.10"
 license = "MIT"
 license-files = ["LICENSE"]
 authors = [{ name = "SpecStream", email = "support@specstream.ai" }]
-keywords = ["haystack", "haystack-ai", "specstream", "agents", "tools", "web-data", "live-web", "lead-lists", "research"]
+keywords = ["haystack", "haystack-ai", "specstream", "agents", "tools", "web-data", "live-web", "lead-lists", "research", "lead-generation", "lead-list", "company-list", "grants"]
 classifiers = [
   "Development Status :: 4 - Beta",
   "Intended Audience :: Developers",
@@ -30,8 +30,8 @@ test = ["pytest>=8", "ruff"]
 [project.urls]
 Homepage = "https://specstream.ai"
 Documentation = "https://specstream.ai/docs"
-Repository = "https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack"
-Issues = "https://github.com/Specstreamai/specstream-tools/issues"
+Repository = "https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack"
+Issues = "https://github.com/specstream/specstream-tools/issues"
 Pricing = "https://specstream.ai/pricing"
 
 [build-system]

{specstream_haystack-0.1.0 → specstream_haystack-0.1.2}/src/haystack_integrations/components/fetchers/specstream/_api.py

@@ -12,7 +12,7 @@ from dataclasses import dataclass
 from typing import Any, Callable
 
 DEFAULT_BASE_URL = "https://api.specstream.ai"
-USER_AGENT = "langchain-specstream/0.1.2"
+USER_AGENT = "specstream-haystack/0.1.2"
 # A list runs for up to about five minutes, so the tools wait a little longer than that by default.
 DEFAULT_WAIT_SECONDS = 420.0
 FINISHED = {"completed", "partial"}
@@ -35,15 +35,23 @@ def list_link(find_id: str) -> str:
     return f"https://specstream.ai/workspace/collections/{urllib.parse.quote(find_id, safe='')}"
 
 
-def flatten_row(find_id: str, record: dict[str, Any]) -> dict[str, Any]:
-    """One row as flat fields: each column's label holds what the list shows, plus where the row was read."""
+def _sources(flat: dict[str, Any]) -> dict[str, Any]:
+    """Each value's page in the API's flat row: <field>_source beside the <field> it belongs to."""
+    return {key: value for key, value in flat.items() if key.endswith("_source") and key[: -len("_source")] in flat}
+
+
+def flatten_row(find_id: str, record: dict[str, Any], flat: dict[str, Any] | None = None) -> dict[str, Any]:
+    """One row as flat fields: each column's label holds what the list shows. The page of each value
+    (<field>_source) and of the fact that put the row in the list (source_url, checked_at) come from
+    the API's flat row, so every tool shows the same pages."""
+    flat = flat or {}
+    sources = _sources(flat)
     row: dict[str, Any] = {"list_id": find_id, "position": record.get("ordinal")}
-    source: dict[str, Any] | None = None
     for cell in record.get("details") or record.get("cells") or []:
         label = str(cell.get("label") or cell.get("column_id") or "").strip()
         if not label:
             continue
-        if label in RESERVED:
+        if label in RESERVED or label in sources:
             label = f"{label} (column)"
         if cell.get("absence"):
             value: Any = ""
@@ -52,11 +60,9 @@ def flatten_row(find_id: str, record: dict[str, Any]) -> dict[str, Any]:
         else:
             value = cell.get("value") if cell.get("value") is not None else ""
         row[label] = value
-        provenance = cell.get("provenance") or {}
-        if source is None and str(provenance.get("source_url") or "").startswith("http"):
-            source = provenance
-    row["source_url"] = source.get("source_url", "") if source else ""
-    row["checked_at"] = source.get("observed_at", "") if source else ""
+    row.update(sources)
+    row["source_url"] = flat.get("source_url") or ""
+    row["checked_at"] = flat.get("checked_at") or ""
     row["list_link"] = list_link(find_id)
     return row
 
@@ -115,7 +121,11 @@ class SpecStreamAPI:
             query = {"limit": "100", **({"cursor": cursor} if cursor else {})}
             page = self.request("GET", f"/v1/finds/{urllib.parse.quote(find_id, safe='')}/rows?{urllib.parse.urlencode(query)}") or {}
             status = page.get("status") or status
-            rows.extend(flatten_row(find_id, record) for record in page.get("rows") or [])
+            flat = {str(item.get("id")): item for item in page.get("flat_rows") or []}
+            rows.extend(
+                flatten_row(find_id, record, flat.get(f"{find_id}:{record.get('entity_id')}"))
+                for record in page.get("rows") or []
+            )
             cursor = page.get("next_cursor")
             if not cursor:
                 break

{specstream_haystack-0.1.0 → specstream_haystack-0.1.2}/tests/test_specstream_haystack.py

@@ -13,6 +13,7 @@ FIND_ID = "fnd_test123"
 
 def _row(ordinal, name, url):
     return {
+        "entity_id": f"e{ordinal}",
         "ordinal": ordinal,
         "details": [
             {"label": "Organization", "display": name, "provenance": {"source_url": url, "observed_at": "2026-10-14T19:56:02Z"}},
@@ -22,6 +23,18 @@ def _row(ordinal, name, url):
     }
 
 
+def _flat(row):
+    """The API's flat row: the page of each value, and of the fact that put the row in the list."""
+    name = row["details"][0]
+    return {
+        "id": f"{FIND_ID}:{row['entity_id']}",
+        "name": name["display"],
+        "name_source": name["provenance"]["source_url"],
+        "source_url": f"{name['provenance']['source_url']}/filing",
+        "checked_at": "2026-10-14T19:56:02Z",
+    }
+
+
 class FakeApi:
     """Stands in for the SpecStream REST API and records the calls."""
 
@@ -43,7 +56,8 @@ class FakeApi:
         if path == "/v1/capacity/executions":
             return {"find_id": FIND_ID}
         if "/rows" in path:
-            return self.pages.pop(0) if len(self.pages) > 1 else self.pages[0]
+            page = self.pages.pop(0) if len(self.pages) > 1 else self.pages[0]
+            return {**page, "flat_rows": [_flat(row) for row in page["rows"]]}
         raise AssertionError(path)
 
     def paths(self):
@@ -71,7 +85,7 @@ def test_rows_become_documents_with_source_in_meta(fake):
     fake()
     result = _fetcher().run(request="Breweries in Portland")
     docs = result["documents"]
-    assert [d.meta["source_url"] for d in docs] == ["https://src/a", "https://src/b"]
+    assert [d.meta["source_url"] for d in docs] == ["https://src/a/filing", "https://src/b/filing"]
     first = docs[0]
     assert json.loads(first.content)["Organization"] == "Alpha"
     assert first.meta["checked_at"] == "2026-10-14T19:56:02Z"