specstream-haystack 0.1.2__tar.gz → 0.2.0__tar.gz

{specstream_haystack-0.1.2 → specstream_haystack-0.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.5
 Name: specstream-haystack
-Version: 0.1.2
-Summary: Give a Haystack pipeline or agent lead lists, company lists and grants from the live web. Every row comes with its source, priced per match in code.
+Version: 0.2.0
+Summary: The live web data engine for Haystack: Search API ($5 per 1,000 searches), Extract API ($0.80 per 1,000 pages) and Deep Search.
 Project-URL: Homepage, https://specstream.ai
 Project-URL: Documentation, https://specstream.ai/docs
 Project-URL: Repository, https://github.com/specstream/specstream-tools/tree/main/packages/specstream-haystack
@@ -33,14 +33,18 @@ Description-Content-Type: text/markdown
 
 # specstream-haystack
 
-Give your Haystack pipeline or agent any list from the live web. Search components hand a pipeline pages to read; SpecStream hands it the finished list. Ask in plain words – "dental clinics in Lisbon with online booking", "US startups that filed a $5M+ Form D this week" – and get back every matching row as a `Document`, each with the page it was read from. The list starts only within a price you set in code, and the model can't raise it.
+[SpecStream](https://specstream.ai) is the live web data engine. One platform to build, search, extract, enrich, map, and monitor data across the entire web.
+
+Ask for any dataset in plain words, and SpecStream builds it from the entire live web: every match found, extracted and enriched into structured records, then monitored for change. This package gives your Haystack pipeline or agent a fetcher component and a tool for it, with one `Document` per row. A run starts only within a price you set in code, and the model can't raise it.
+
+Source URL and read time on every value.
 
 ```bash
 pip install -U specstream-haystack
 export SPECSTREAM_API_KEY=ss_live_...   # Agents & API in your SpecStream workspace
 ```
 
-A free account comes with $10 of work to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
+A free account comes with $10 of credit to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
 
 ## In a pipeline
 

{specstream_haystack-0.1.2 → specstream_haystack-0.2.0}/README.md

@@ -1,13 +1,17 @@
 # specstream-haystack
 
-Give your Haystack pipeline or agent any list from the live web. Search components hand a pipeline pages to read; SpecStream hands it the finished list. Ask in plain words – "dental clinics in Lisbon with online booking", "US startups that filed a $5M+ Form D this week" – and get back every matching row as a `Document`, each with the page it was read from. The list starts only within a price you set in code, and the model can't raise it.
+[SpecStream](https://specstream.ai) is the live web data engine. One platform to build, search, extract, enrich, map, and monitor data across the entire web.
+
+Ask for any dataset in plain words, and SpecStream builds it from the entire live web: every match found, extracted and enriched into structured records, then monitored for change. This package gives your Haystack pipeline or agent a fetcher component and a tool for it, with one `Document` per row. A run starts only within a price you set in code, and the model can't raise it.
+
+Source URL and read time on every value.
 
 ```bash
 pip install -U specstream-haystack
 export SPECSTREAM_API_KEY=ss_live_...   # Agents & API in your SpecStream workspace
 ```
 
-A free account comes with $10 of work to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
+A free account comes with $10 of credit to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
 
 ## In a pipeline
 

{specstream_haystack-0.1.2 → specstream_haystack-0.2.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "specstream-haystack"
-version = "0.1.2"
-description = "Give a Haystack pipeline or agent lead lists, company lists and grants from the live web. Every row comes with its source, priced per match in code."
+version = "0.2.0"
+description = "The live web data engine for Haystack: Search API ($5 per 1,000 searches), Extract API ($0.80 per 1,000 pages) and Deep Search."
 readme = "README.md"
 requires-python = ">=3.10"
 license = "MIT"

specstream_haystack-0.2.0/src/haystack_integrations/components/fetchers/specstream/__init__.py

@@ -0,0 +1,8 @@
+from haystack_integrations.components.fetchers.specstream._api import SpecStreamError
+from haystack_integrations.components.fetchers.specstream.list_fetcher import (
+    SpecStreamExtractor,
+    SpecStreamListFetcher,
+    SpecStreamWebSearch,
+)
+
+__all__ = ["SpecStreamError", "SpecStreamExtractor", "SpecStreamListFetcher", "SpecStreamWebSearch"]

{specstream_haystack-0.1.2 → specstream_haystack-0.2.0}/src/haystack_integrations/components/fetchers/specstream/_api.py

@@ -112,6 +112,24 @@ class SpecStreamAPI:
             raise SpecStreamError("SpecStream returned a response this package couldn't read.")
         return payload["data"]
 
+
+    def web_search(self, q: str, *, passages: bool = False) -> dict[str, Any]:
+        """Search the live web with GET /v1/web-search ($5 per 1,000 searches, no quote needed)."""
+        query = (q or "").strip()
+        if not query or len(query) > 2000:
+            raise ValueError("q must be between 1 and 2,000 characters.")
+        params: dict[str, str] = {"q": query}
+        if passages:
+            params["passages"] = "true"
+        return self.request("GET", f"/v1/web-search?{urllib.parse.urlencode(params)}") or {}
+
+    def extract_pages(self, urls: list[str], *, format: str = "markdown") -> dict[str, Any]:
+        """Extract the main text of 1 to 20 URLs live with POST /v1/extract ($0.80 per 1,000 pages)."""
+        cleaned = [str(u or "").strip() for u in (urls or []) if str(u or "").strip()]
+        if not 1 <= len(cleaned) <= 20:
+            raise ValueError("Provide 1 to 20 http or https URLs.")
+        fmt = "text" if format == "text" else "markdown"
+        return self.request("POST", "/v1/extract", {"urls": cleaned, "format": fmt}) or {}
     def results(self, find_id: str) -> dict[str, Any]:
         """Every row found so far for a list, following the pages."""
         rows: list[dict[str, Any]] = []

{specstream_haystack-0.1.2 → specstream_haystack-0.2.0}/src/haystack_integrations/components/fetchers/specstream/list_fetcher.py

@@ -119,3 +119,86 @@ class SpecStreamListFetcher:
         meta["request"] = request
         meta["fields"] = fields
         return Document(content=json.dumps(fields, ensure_ascii=False, default=str), meta=meta)
+
+
+@component
+class SpecStreamWebSearch:
+    """Searches the live web in seconds with the SpecStream Search API (`GET /v1/web-search`, $5 per 1,000 searches)."""
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SPECSTREAM_API_KEY"),
+        base_url: str = DEFAULT_BASE_URL,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(self, api_key=self.api_key.to_dict(), base_url=self.base_url)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SpecStreamWebSearch":
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document], answer=dict[str, Any])
+    def run(self, q: str, passages: bool = False) -> dict[str, Any]:
+        data = SpecStreamAPI(self.api_key.resolve_value() or "", self.base_url).web_search(q, passages=passages)
+        docs: list[Document] = []
+        for row in data.get("rows") or []:
+            if isinstance(row, dict):
+                docs.append(
+                    Document(
+                        content=str(row.get("snippet") or row.get("name") or ""),
+                        meta={"title": row.get("name"), "source_url": row.get("page"), "fields": row.get("fields")},
+                    )
+                )
+        for src in data.get("sources") or []:
+            if isinstance(src, dict):
+                docs.append(
+                    Document(
+                        content=str(src.get("snippet") or src.get("title") or ""),
+                        meta={"title": src.get("title"), "source_url": src.get("url"), "published": src.get("published")},
+                    )
+                )
+        return {"documents": docs, "answer": data.get("answer") or {}}
+
+
+@component
+class SpecStreamExtractor:
+    """Reads the main text of 1 to 20 URLs live with the SpecStream Extract API (`POST /v1/extract`, $0.80 per 1,000 pages)."""
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SPECSTREAM_API_KEY"),
+        base_url: str = DEFAULT_BASE_URL,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(self, api_key=self.api_key.to_dict(), base_url=self.base_url)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SpecStreamExtractor":
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document])
+    def run(self, urls: list[str], format: str = "markdown") -> dict[str, Any]:
+        data = SpecStreamAPI(self.api_key.resolve_value() or "", self.base_url).extract_pages(urls, format=format)
+        docs = [
+            Document(
+                content=str(item.get("text") or ""),
+                meta={
+                    "source_url": item.get("url"),
+                    "title": item.get("title"),
+                    "published": item.get("published"),
+                    "status": item.get("status"),
+                    "read_from": item.get("read_from"),
+                },
+            )
+            for item in (data.get("results") or [])
+            if isinstance(item, dict)
+        ]
+        return {"documents": docs}

specstream_haystack-0.1.2/src/haystack_integrations/components/fetchers/specstream/__init__.py

@@ -1,4 +0,0 @@
-from haystack_integrations.components.fetchers.specstream._api import SpecStreamError
-from haystack_integrations.components.fetchers.specstream.list_fetcher import SpecStreamListFetcher
-
-__all__ = ["SpecStreamError", "SpecStreamListFetcher"]