specstream-haystack 0.1.1__tar.gz → 0.2.0__tar.gz

{specstream_haystack-0.1.1 → specstream_haystack-0.2.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.5
 Name: specstream-haystack
-Version: 0.1.1
-Summary: Give a Haystack pipeline or agent lead lists, company lists and grants from the live web. Every row comes with its source, priced per match in code.
+Version: 0.2.0
+Summary: The live web data engine for Haystack: Search API ($5 per 1,000 searches), Extract API ($0.80 per 1,000 pages) and Deep Search.
 Project-URL: Homepage, https://specstream.ai
 Project-URL: Documentation, https://specstream.ai/docs
-Project-URL: Repository, https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack
-Project-URL: Issues, https://github.com/Specstreamai/specstream-tools/issues
+Project-URL: Repository, https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack
+Project-URL: Issues, https://github.com/specstream/specstream-tools/issues
 Project-URL: Pricing, https://specstream.ai/pricing
 Author-email: SpecStream <support@specstream.ai>
 License-Expression: MIT
@@ -33,14 +33,18 @@ Description-Content-Type: text/markdown
 
 # specstream-haystack
 
-Give your Haystack pipeline or agent any list from the live web. Search components hand a pipeline pages to read; SpecStream hands it the finished list. Ask in plain words – "dental clinics in Lisbon with online booking", "US startups that filed a $5M+ Form D this week" – and get back every matching row as a `Document`, each with the page it was read from. The list starts only within a price you set in code, and the model can't raise it.
+[SpecStream](https://specstream.ai) is the live web data engine. One platform to build, search, extract, enrich, map, and monitor data across the entire web.
+
+Ask for any dataset in plain words, and SpecStream builds it from the entire live web: every match found, extracted and enriched into structured records, then monitored for change. This package gives your Haystack pipeline or agent a fetcher component and a tool for it, with one `Document` per row. A run starts only within a price you set in code, and the model can't raise it.
+
+Source URL and read time on every value.
 
 ```bash
 pip install -U specstream-haystack
 export SPECSTREAM_API_KEY=ss_live_...   # Agents & API in your SpecStream workspace
 ```
 
-A free account comes with $10 of work to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
+A free account comes with $10 of credit to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
 
 ## In a pipeline
 
@@ -80,4 +84,4 @@ The agent writes the request and picks how many rows. You decide what one list m
 
 The same request with the same limits, in the same hour, reaches the run it already started instead of starting another. A retried step never pays for the list twice.
 
-MIT licensed. Source: [Specstreamai/specstream-tools](https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack).
+MIT licensed. Source: [specstream/specstream-tools](https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack).

{specstream_haystack-0.1.1 → specstream_haystack-0.2.0}/README.md

@@ -1,13 +1,17 @@
 # specstream-haystack
 
-Give your Haystack pipeline or agent any list from the live web. Search components hand a pipeline pages to read; SpecStream hands it the finished list. Ask in plain words – "dental clinics in Lisbon with online booking", "US startups that filed a $5M+ Form D this week" – and get back every matching row as a `Document`, each with the page it was read from. The list starts only within a price you set in code, and the model can't raise it.
+[SpecStream](https://specstream.ai) is the live web data engine. One platform to build, search, extract, enrich, map, and monitor data across the entire web.
+
+Ask for any dataset in plain words, and SpecStream builds it from the entire live web: every match found, extracted and enriched into structured records, then monitored for change. This package gives your Haystack pipeline or agent a fetcher component and a tool for it, with one `Document` per row. A run starts only within a price you set in code, and the model can't raise it.
+
+Source URL and read time on every value.
 
 ```bash
 pip install -U specstream-haystack
 export SPECSTREAM_API_KEY=ss_live_...   # Agents & API in your SpecStream workspace
 ```
 
-A free account comes with $10 of work to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
+A free account comes with $10 of credit to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
 
 ## In a pipeline
 
@@ -47,4 +51,4 @@ The agent writes the request and picks how many rows. You decide what one list m
 
 The same request with the same limits, in the same hour, reaches the run it already started instead of starting another. A retried step never pays for the list twice.
 
-MIT licensed. Source: [Specstreamai/specstream-tools](https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack).
+MIT licensed. Source: [specstream/specstream-tools](https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack).

{specstream_haystack-0.1.1 → specstream_haystack-0.2.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "specstream-haystack"
-version = "0.1.1"
-description = "Give a Haystack pipeline or agent lead lists, company lists and grants from the live web. Every row comes with its source, priced per match in code."
+version = "0.2.0"
+description = "The live web data engine for Haystack: Search API ($5 per 1,000 searches), Extract API ($0.80 per 1,000 pages) and Deep Search."
 readme = "README.md"
 requires-python = ">=3.10"
 license = "MIT"
@@ -30,8 +30,8 @@ test = ["pytest>=8", "ruff"]
 [project.urls]
 Homepage = "https://specstream.ai"
 Documentation = "https://specstream.ai/docs"
-Repository = "https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack"
-Issues = "https://github.com/Specstreamai/specstream-tools/issues"
+Repository = "https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack"
+Issues = "https://github.com/specstream/specstream-tools/issues"
 Pricing = "https://specstream.ai/pricing"
 
 [build-system]

specstream_haystack-0.2.0/src/haystack_integrations/components/fetchers/specstream/__init__.py

@@ -0,0 +1,8 @@
+from haystack_integrations.components.fetchers.specstream._api import SpecStreamError
+from haystack_integrations.components.fetchers.specstream.list_fetcher import (
+    SpecStreamExtractor,
+    SpecStreamListFetcher,
+    SpecStreamWebSearch,
+)
+
+__all__ = ["SpecStreamError", "SpecStreamExtractor", "SpecStreamListFetcher", "SpecStreamWebSearch"]

{specstream_haystack-0.1.1 → specstream_haystack-0.2.0}/src/haystack_integrations/components/fetchers/specstream/_api.py

@@ -12,7 +12,7 @@ from dataclasses import dataclass
 from typing import Any, Callable
 
 DEFAULT_BASE_URL = "https://api.specstream.ai"
-USER_AGENT = "specstream-haystack/0.1.1"
+USER_AGENT = "specstream-haystack/0.1.2"
 # A list runs for up to about five minutes, so the tools wait a little longer than that by default.
 DEFAULT_WAIT_SECONDS = 420.0
 FINISHED = {"completed", "partial"}
@@ -35,15 +35,23 @@ def list_link(find_id: str) -> str:
     return f"https://specstream.ai/workspace/collections/{urllib.parse.quote(find_id, safe='')}"
 
 
-def flatten_row(find_id: str, record: dict[str, Any]) -> dict[str, Any]:
-    """One row as flat fields: each column's label holds what the list shows, plus where the row was read."""
+def _sources(flat: dict[str, Any]) -> dict[str, Any]:
+    """Each value's page in the API's flat row: <field>_source beside the <field> it belongs to."""
+    return {key: value for key, value in flat.items() if key.endswith("_source") and key[: -len("_source")] in flat}
+
+
+def flatten_row(find_id: str, record: dict[str, Any], flat: dict[str, Any] | None = None) -> dict[str, Any]:
+    """One row as flat fields: each column's label holds what the list shows. The page of each value
+    (<field>_source) and of the fact that put the row in the list (source_url, checked_at) come from
+    the API's flat row, so every tool shows the same pages."""
+    flat = flat or {}
+    sources = _sources(flat)
     row: dict[str, Any] = {"list_id": find_id, "position": record.get("ordinal")}
-    source: dict[str, Any] | None = None
     for cell in record.get("details") or record.get("cells") or []:
         label = str(cell.get("label") or cell.get("column_id") or "").strip()
         if not label:
             continue
-        if label in RESERVED:
+        if label in RESERVED or label in sources:
             label = f"{label} (column)"
         if cell.get("absence"):
             value: Any = ""
@@ -52,11 +60,9 @@ def flatten_row(find_id: str, record: dict[str, Any]) -> dict[str, Any]:
         else:
             value = cell.get("value") if cell.get("value") is not None else ""
         row[label] = value
-        provenance = cell.get("provenance") or {}
-        if source is None and str(provenance.get("source_url") or "").startswith("http"):
-            source = provenance
-    row["source_url"] = source.get("source_url", "") if source else ""
-    row["checked_at"] = source.get("observed_at", "") if source else ""
+    row.update(sources)
+    row["source_url"] = flat.get("source_url") or ""
+    row["checked_at"] = flat.get("checked_at") or ""
     row["list_link"] = list_link(find_id)
     return row
 
@@ -106,6 +112,24 @@ class SpecStreamAPI:
             raise SpecStreamError("SpecStream returned a response this package couldn't read.")
         return payload["data"]
 
+
+    def web_search(self, q: str, *, passages: bool = False) -> dict[str, Any]:
+        """Search the live web with GET /v1/web-search ($5 per 1,000 searches, no quote needed)."""
+        query = (q or "").strip()
+        if not query or len(query) > 2000:
+            raise ValueError("q must be between 1 and 2,000 characters.")
+        params: dict[str, str] = {"q": query}
+        if passages:
+            params["passages"] = "true"
+        return self.request("GET", f"/v1/web-search?{urllib.parse.urlencode(params)}") or {}
+
+    def extract_pages(self, urls: list[str], *, format: str = "markdown") -> dict[str, Any]:
+        """Extract the main text of 1 to 20 URLs live with POST /v1/extract ($0.80 per 1,000 pages)."""
+        cleaned = [str(u or "").strip() for u in (urls or []) if str(u or "").strip()]
+        if not 1 <= len(cleaned) <= 20:
+            raise ValueError("Provide 1 to 20 http or https URLs.")
+        fmt = "text" if format == "text" else "markdown"
+        return self.request("POST", "/v1/extract", {"urls": cleaned, "format": fmt}) or {}
     def results(self, find_id: str) -> dict[str, Any]:
         """Every row found so far for a list, following the pages."""
         rows: list[dict[str, Any]] = []
@@ -115,7 +139,11 @@ class SpecStreamAPI:
             query = {"limit": "100", **({"cursor": cursor} if cursor else {})}
             page = self.request("GET", f"/v1/finds/{urllib.parse.quote(find_id, safe='')}/rows?{urllib.parse.urlencode(query)}") or {}
             status = page.get("status") or status
-            rows.extend(flatten_row(find_id, record) for record in page.get("rows") or [])
+            flat = {str(item.get("id")): item for item in page.get("flat_rows") or []}
+            rows.extend(
+                flatten_row(find_id, record, flat.get(f"{find_id}:{record.get('entity_id')}"))
+                for record in page.get("rows") or []
+            )
             cursor = page.get("next_cursor")
             if not cursor:
                 break

{specstream_haystack-0.1.1 → specstream_haystack-0.2.0}/src/haystack_integrations/components/fetchers/specstream/list_fetcher.py

@@ -119,3 +119,86 @@ class SpecStreamListFetcher:
         meta["request"] = request
         meta["fields"] = fields
         return Document(content=json.dumps(fields, ensure_ascii=False, default=str), meta=meta)
+
+
+@component
+class SpecStreamWebSearch:
+    """Searches the live web in seconds with the SpecStream Search API (`GET /v1/web-search`, $5 per 1,000 searches)."""
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SPECSTREAM_API_KEY"),
+        base_url: str = DEFAULT_BASE_URL,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(self, api_key=self.api_key.to_dict(), base_url=self.base_url)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SpecStreamWebSearch":
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document], answer=dict[str, Any])
+    def run(self, q: str, passages: bool = False) -> dict[str, Any]:
+        data = SpecStreamAPI(self.api_key.resolve_value() or "", self.base_url).web_search(q, passages=passages)
+        docs: list[Document] = []
+        for row in data.get("rows") or []:
+            if isinstance(row, dict):
+                docs.append(
+                    Document(
+                        content=str(row.get("snippet") or row.get("name") or ""),
+                        meta={"title": row.get("name"), "source_url": row.get("page"), "fields": row.get("fields")},
+                    )
+                )
+        for src in data.get("sources") or []:
+            if isinstance(src, dict):
+                docs.append(
+                    Document(
+                        content=str(src.get("snippet") or src.get("title") or ""),
+                        meta={"title": src.get("title"), "source_url": src.get("url"), "published": src.get("published")},
+                    )
+                )
+        return {"documents": docs, "answer": data.get("answer") or {}}
+
+
+@component
+class SpecStreamExtractor:
+    """Reads the main text of 1 to 20 URLs live with the SpecStream Extract API (`POST /v1/extract`, $0.80 per 1,000 pages)."""
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SPECSTREAM_API_KEY"),
+        base_url: str = DEFAULT_BASE_URL,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(self, api_key=self.api_key.to_dict(), base_url=self.base_url)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SpecStreamExtractor":
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document])
+    def run(self, urls: list[str], format: str = "markdown") -> dict[str, Any]:
+        data = SpecStreamAPI(self.api_key.resolve_value() or "", self.base_url).extract_pages(urls, format=format)
+        docs = [
+            Document(
+                content=str(item.get("text") or ""),
+                meta={
+                    "source_url": item.get("url"),
+                    "title": item.get("title"),
+                    "published": item.get("published"),
+                    "status": item.get("status"),
+                    "read_from": item.get("read_from"),
+                },
+            )
+            for item in (data.get("results") or [])
+            if isinstance(item, dict)
+        ]
+        return {"documents": docs}

{specstream_haystack-0.1.1 → specstream_haystack-0.2.0}/tests/test_specstream_haystack.py

@@ -13,6 +13,7 @@ FIND_ID = "fnd_test123"
 
 def _row(ordinal, name, url):
     return {
+        "entity_id": f"e{ordinal}",
         "ordinal": ordinal,
         "details": [
             {"label": "Organization", "display": name, "provenance": {"source_url": url, "observed_at": "2026-10-14T19:56:02Z"}},
@@ -22,6 +23,18 @@ def _row(ordinal, name, url):
     }
 
 
+def _flat(row):
+    """The API's flat row: the page of each value, and of the fact that put the row in the list."""
+    name = row["details"][0]
+    return {
+        "id": f"{FIND_ID}:{row['entity_id']}",
+        "name": name["display"],
+        "name_source": name["provenance"]["source_url"],
+        "source_url": f"{name['provenance']['source_url']}/filing",
+        "checked_at": "2026-10-14T19:56:02Z",
+    }
+
+
 class FakeApi:
     """Stands in for the SpecStream REST API and records the calls."""
 
@@ -43,7 +56,8 @@ class FakeApi:
         if path == "/v1/capacity/executions":
             return {"find_id": FIND_ID}
         if "/rows" in path:
-            return self.pages.pop(0) if len(self.pages) > 1 else self.pages[0]
+            page = self.pages.pop(0) if len(self.pages) > 1 else self.pages[0]
+            return {**page, "flat_rows": [_flat(row) for row in page["rows"]]}
         raise AssertionError(path)
 
     def paths(self):
@@ -71,7 +85,7 @@ def test_rows_become_documents_with_source_in_meta(fake):
     fake()
     result = _fetcher().run(request="Breweries in Portland")
     docs = result["documents"]
-    assert [d.meta["source_url"] for d in docs] == ["https://src/a", "https://src/b"]
+    assert [d.meta["source_url"] for d in docs] == ["https://src/a/filing", "https://src/b/filing"]
     first = docs[0]
     assert json.loads(first.content)["Organization"] == "Alpha"
     assert first.meta["checked_at"] == "2026-10-14T19:56:02Z"

specstream_haystack-0.1.1/src/haystack_integrations/components/fetchers/specstream/__init__.py

@@ -1,4 +0,0 @@
-from haystack_integrations.components.fetchers.specstream._api import SpecStreamError
-from haystack_integrations.components.fetchers.specstream.list_fetcher import SpecStreamListFetcher
-
-__all__ = ["SpecStreamError", "SpecStreamListFetcher"]