specstream-haystack 0.1.0__py3-none-any.whl

haystack_integrations/components/fetchers/specstream/__init__.py

@@ -0,0 +1,4 @@
+from haystack_integrations.components.fetchers.specstream._api import SpecStreamError
+from haystack_integrations.components.fetchers.specstream.list_fetcher import SpecStreamListFetcher
+
+__all__ = ["SpecStreamError", "SpecStreamListFetcher"]

haystack_integrations/components/fetchers/specstream/_api.py

@@ -0,0 +1,217 @@
+"""The SpecStream REST calls the tools share: read a request, price it, start it within a maximum, read its rows."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Any, Callable
+
+DEFAULT_BASE_URL = "https://api.specstream.ai"
+USER_AGENT = "langchain-specstream/0.1.2"
+# A list runs for up to about five minutes, so the tools wait a little longer than that by default.
+DEFAULT_WAIT_SECONDS = 420.0
+FINISHED = {"completed", "partial"}
+STOPPED = {"failed", "cancelled", "canceled", "expired"}
+# Fields every row carries; a column with the same name is renamed so it can't overwrite them.
+RESERVED = {"list_id", "list_link", "position", "source_url", "checked_at"}
+
+Opener = Callable[[urllib.request.Request, float], Any]
+
+
+class SpecStreamError(RuntimeError):
+    """SpecStream refused a call or answered with something this package can't read."""
+
+
+def _dollars(cents: int) -> str:
+    return f"${cents / 100:.2f}"
+
+
+def list_link(find_id: str) -> str:
+    return f"https://specstream.ai/workspace/collections/{urllib.parse.quote(find_id, safe='')}"
+
+
+def flatten_row(find_id: str, record: dict[str, Any]) -> dict[str, Any]:
+    """One row as flat fields: each column's label holds what the list shows, plus where the row was read."""
+    row: dict[str, Any] = {"list_id": find_id, "position": record.get("ordinal")}
+    source: dict[str, Any] | None = None
+    for cell in record.get("details") or record.get("cells") or []:
+        label = str(cell.get("label") or cell.get("column_id") or "").strip()
+        if not label:
+            continue
+        if label in RESERVED:
+            label = f"{label} (column)"
+        if cell.get("absence"):
+            value: Any = ""
+        elif cell.get("display") is not None:
+            value = cell["display"]
+        else:
+            value = cell.get("value") if cell.get("value") is not None else ""
+        row[label] = value
+        provenance = cell.get("provenance") or {}
+        if source is None and str(provenance.get("source_url") or "").startswith("http"):
+            source = provenance
+    row["source_url"] = source.get("source_url", "") if source else ""
+    row["checked_at"] = source.get("observed_at", "") if source else ""
+    row["list_link"] = list_link(find_id)
+    return row
+
+
+@dataclass
+class SpecStreamAPI:
+    api_key: str
+    base_url: str = DEFAULT_BASE_URL
+    timeout: float = 150.0
+    opener: Opener | None = None
+    sleep: Callable[[float], None] = time.sleep
+    clock: Callable[[], float] = time.monotonic
+
+    def request(self, method: str, path: str, body: dict[str, Any] | None = None, *, allow_404: bool = False) -> dict[str, Any] | None:
+        key = (self.api_key or "").strip()
+        if not key:
+            raise SpecStreamError("Set SPECSTREAM_API_KEY or pass api_key. Create a key under Agents & API in your SpecStream workspace.")
+        request = urllib.request.Request(
+            self.base_url.rstrip("/") + path,
+            data=None if body is None else json.dumps(body).encode(),
+            method=method,
+            headers={
+                "Authorization": key if key.lower().startswith("bearer ") else f"Bearer {key}",
+                "Accept": "application/json",
+                "Content-Type": "application/json",
+                "User-Agent": USER_AGENT,
+            },
+        )
+        open_ = self.opener or (lambda req, timeout: urllib.request.urlopen(req, timeout=timeout))
+        try:
+            response = open_(request, self.timeout)
+            try:
+                raw = response.read()
+            finally:
+                response.close()
+        except urllib.error.HTTPError as error:
+            if allow_404 and error.code == 404:
+                return None
+            raise SpecStreamError(_error_message(error)) from None
+        except urllib.error.URLError as error:
+            raise SpecStreamError(f"Couldn't reach SpecStream: {error.reason}") from None
+        try:
+            payload = json.loads(raw or b"{}")
+        except ValueError:
+            payload = None
+        if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
+            raise SpecStreamError("SpecStream returned a response this package couldn't read.")
+        return payload["data"]
+
+    def results(self, find_id: str) -> dict[str, Any]:
+        """Every row found so far for a list, following the pages."""
+        rows: list[dict[str, Any]] = []
+        cursor: str | None = None
+        status = "running"
+        while True:
+            query = {"limit": "100", **({"cursor": cursor} if cursor else {})}
+            page = self.request("GET", f"/v1/finds/{urllib.parse.quote(find_id, safe='')}/rows?{urllib.parse.urlencode(query)}") or {}
+            status = page.get("status") or status
+            rows.extend(flatten_row(find_id, record) for record in page.get("rows") or [])
+            cursor = page.get("next_cursor")
+            if not cursor:
+                break
+        return {"list_id": find_id, "status": status, "list_link": list_link(find_id), "row_count": len(rows), "rows": rows}
+
+    def find_list(
+        self,
+        request: str,
+        *,
+        max_results: int,
+        max_price_usd: float,
+        wait_seconds: float = DEFAULT_WAIT_SECONDS,
+        poll_interval: float = 3.0,
+        locale: str = "en-US",
+        key_prefix: str = "haystack",
+    ) -> dict[str, Any]:
+        q = (request or "").strip()
+        if not q:
+            raise ValueError("Describe the list to find.")
+        if not isinstance(max_results, int) or max_results < 1:
+            raise ValueError("max_results must be 1 or more.")
+        max_cents = round(float(max_price_usd) * 100)
+        if max_cents <= 0:
+            raise ValueError("max_price_usd must be greater than 0.")
+
+        # The same request with the same limits gets the run it already started in this hour or the one before,
+        # so a retried agent step never buys the list twice, even across the turn of the hour.
+        def key_for(at: float) -> str:
+            hour = time.strftime("%Y-%m-%dT%H", time.gmtime(at))
+            return f"{key_prefix}-" + hashlib.sha256(f"{q}|{max_results}|{max_cents}|{hour}".encode()).hexdigest()[:32]
+
+        now = time.time()
+        request_key = key_for(now)
+        existing = None
+        for candidate in (request_key, key_for(now - 3600)):
+            existing = self.request("GET", f"/v1/capacity/requests/{urllib.parse.quote(candidate, safe='')}", allow_404=True)
+            if existing and existing.get("find_id"):
+                break
+        if existing and existing.get("find_id"):
+            find_id = existing["find_id"]
+            maximum_cents = int(existing.get("maximum_cents") or max_cents)
+        else:
+            reading = self.request("POST", "/v1/parse", {"q": q, "locale": locale}) or {}
+            if reading.get("status") != "runnable":
+                read_as = reading.get("canonical_sentence") or q
+                return {
+                    "status": "needs_edit",
+                    "request": q,
+                    "message": f'SpecStream needs a clearer request before it can price it. It read: "{read_as}". Add the place or what every row must meet.',
+                }
+            quote = (
+                self.request(
+                    "POST", "/v1/capacity/quotes", {"ast_hash": reading["ast_hash"], "operation": "find", "requested_units": max_results}
+                )
+                or {}
+            )
+            maximum_cents = int(quote.get("maximum_cents") or 0)
+            if maximum_cents > max_cents:
+                return {
+                    "status": "over_maximum",
+                    "request": q,
+                    "maximum_price_usd": maximum_cents / 100,
+                    "message": f"Nothing ran: this list can cost up to {_dollars(maximum_cents)}, above the maximum of {_dollars(max_cents)}. Ask for fewer results or raise max_price_usd.",
+                }
+            run = (
+                self.request(
+                    "POST",
+                    "/v1/capacity/executions",
+                    {
+                        "quote_id": quote["quote_id"],
+                        "quote_hash": quote["quote_hash"],
+                        "confirm_max_cents": maximum_cents,
+                        "idempotency_key": request_key,
+                    },
+                )
+                or {}
+            )
+            find_id = run["find_id"]
+
+        deadline = self.clock() + max(0.0, wait_seconds)
+        while True:
+            result = self.results(find_id)
+            if result["status"] in FINISHED | STOPPED or self.clock() >= deadline:
+                break
+            self.sleep(poll_interval)
+        result.update({"request": q, "maximum_price_usd": maximum_cents / 100})
+        if result["status"] not in FINISHED | STOPPED:
+            result["message"] = "Still running. Run the same request again, or open list_link, for the rest."
+        return result
+
+
+def _error_message(error: urllib.error.HTTPError) -> str:
+    try:
+        payload = json.loads(error.read() or b"{}")
+    except ValueError:
+        payload = {}
+    detail = payload.get("error") if isinstance(payload, dict) else None
+    message = detail.get("message") if isinstance(detail, dict) else detail if isinstance(detail, str) else None
+    return f"SpecStream answered {error.code}: {message or error.reason}"

haystack_integrations/components/fetchers/specstream/list_fetcher.py

@@ -0,0 +1,121 @@
+import json
+from typing import Any, Optional
+
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+from haystack_integrations.components.fetchers.specstream._api import (
+    DEFAULT_BASE_URL,
+    DEFAULT_WAIT_SECONDS,
+    SpecStreamAPI,
+    SpecStreamError,
+)
+
+# Row fields that say where a row came from rather than what it says.
+_META = ("list_id", "list_link", "position", "source_url", "checked_at")
+
+
+@component
+class SpecStreamListFetcher:
+    """
+    Turns a plain-English request into the finished list: one Document per matching row.
+
+    The Document content is the row as JSON. The metadata carries the page the row was read from
+    (`source_url`), when it was checked, the list it belongs to, and the row's fields.
+    The list starts only if its price is within `max_price_usd`, which you set in code.
+
+    ```python
+    from haystack_integrations.components.fetchers.specstream import SpecStreamListFetcher
+
+    fetcher = SpecStreamListFetcher(max_price_usd=3, max_results=25)
+    result = fetcher.run(request="Breweries in Portland, Oregon that host events, with their website")
+    for doc in result["documents"]:
+        print(doc.meta["source_url"], doc.content)
+    ```
+    """
+
+    def __init__(
+        self,
+        max_price_usd: float,
+        api_key: Secret = Secret.from_env_var("SPECSTREAM_API_KEY"),
+        max_results: int = 25,
+        base_url: str = DEFAULT_BASE_URL,
+        wait_seconds: float = DEFAULT_WAIT_SECONDS,
+        poll_interval: float = 3.0,
+        locale: str = "en-US",
+    ):
+        """
+        :param max_price_usd: The most one list may cost, in US dollars. If the quoted price is higher, nothing runs.
+        :param api_key: A SpecStream API key. Read from `SPECSTREAM_API_KEY` by default.
+        :param max_results: How many matching rows to find when `run` doesn't say.
+        :param base_url: The SpecStream API address.
+        :param wait_seconds: How long to wait for the list to finish before returning the rows found so far.
+        :param poll_interval: Seconds between checks while the list runs.
+        :param locale: The language the request is read in.
+        """
+        if not max_price_usd or max_price_usd <= 0:
+            raise ValueError("max_price_usd must be greater than 0.")
+        if max_results < 1:
+            raise ValueError("max_results must be 1 or more.")
+        self.max_price_usd = max_price_usd
+        self.api_key = api_key
+        self.max_results = max_results
+        self.base_url = base_url
+        self.wait_seconds = wait_seconds
+        self.poll_interval = poll_interval
+        self.locale = locale
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(
+            self,
+            max_price_usd=self.max_price_usd,
+            api_key=self.api_key.to_dict(),
+            max_results=self.max_results,
+            base_url=self.base_url,
+            wait_seconds=self.wait_seconds,
+            poll_interval=self.poll_interval,
+            locale=self.locale,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SpecStreamListFetcher":
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=list[Document], list_id=str, status=str, list_link=str, message=str)
+    def run(self, request: str, max_results: Optional[int] = None) -> dict[str, Any]:
+        """
+        Find the list and return its rows.
+
+        :param request: The list to find, in plain words: what every row must be or meet, and the details to bring back.
+        :param max_results: How many matching rows to find. Defaults to the value set at init.
+        :returns: `documents` (one per row), `list_id`, `status`, `list_link` and, when the list was still running, a `message`.
+        :raises SpecStreamError: When the request is too vague to price, or the price is above `max_price_usd`.
+        """
+        result = SpecStreamAPI(self.api_key.resolve_value() or "", self.base_url).find_list(
+            request,
+            max_results=max_results or self.max_results,
+            max_price_usd=self.max_price_usd,
+            wait_seconds=self.wait_seconds,
+            poll_interval=self.poll_interval,
+            locale=self.locale,
+            key_prefix="haystack",
+        )
+        if result["status"] in ("needs_edit", "over_maximum"):
+            raise SpecStreamError(result["message"])
+        documents = [self._to_document(row, result.get("request")) for row in result["rows"]]
+        return {
+            "documents": documents,
+            "list_id": result["list_id"],
+            "status": result["status"],
+            "list_link": result["list_link"],
+            "message": result.get("message", ""),
+        }
+
+    @staticmethod
+    def _to_document(row: dict[str, Any], request: Optional[str]) -> Document:
+        fields = {label: value for label, value in row.items() if label not in _META}
+        meta: dict[str, Any] = {key: row.get(key) for key in _META}
+        meta["request"] = request
+        meta["fields"] = fields
+        return Document(content=json.dumps(fields, ensure_ascii=False, default=str), meta=meta)

haystack_integrations/tools/specstream/__init__.py

@@ -0,0 +1,3 @@
+from haystack_integrations.tools.specstream.list_tool import SpecStreamListTool
+
+__all__ = ["SpecStreamListTool"]

haystack_integrations/tools/specstream/list_tool.py

@@ -0,0 +1,50 @@
+from typing import Any
+
+from haystack.tools import ComponentTool
+from haystack.utils import Secret
+
+from haystack_integrations.components.fetchers.specstream import SpecStreamListFetcher
+
+DESCRIPTION = (
+    "Find a list of real organizations, people, places or records on the live web from a plain-English request, "
+    "and get back every matching row with the page it was read from. Use it when the answer is many items that "
+    "meet conditions (companies in a city, clinics with a service, funding filings, open grants, tenders, "
+    "repositories), not a single fact. Results cost money: the list starts only if its price is within the "
+    "maximum the developer set, and only matching rows are paid for."
+)
+
+
+def _rows_as_text(documents: list) -> str:
+    return "\n".join(f"{doc.content}  (source: {doc.meta.get('source_url') or 'n/a'})" for doc in documents)
+
+
+class SpecStreamListTool(ComponentTool):
+    """
+    A Haystack tool an agent can call to get a finished list from the live web.
+
+    The model writes the request and picks how many rows. The price ceiling is set here, in code, and the model can't raise it.
+
+    ```python
+    from haystack.components.agents import Agent
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.dataclasses import ChatMessage
+    from haystack_integrations.tools.specstream import SpecStreamListTool
+
+    agent = Agent(chat_generator=OpenAIChatGenerator(model="gpt-4o"), tools=[SpecStreamListTool(max_price_usd=5)])
+    ```
+    """
+
+    def __init__(
+        self,
+        max_price_usd: float,
+        api_key: Secret = Secret.from_env_var("SPECSTREAM_API_KEY"),
+        name: str = "specstream_find_list",
+        description: str = DESCRIPTION,
+        **fetcher_kwargs: Any,
+    ):
+        super().__init__(
+            component=SpecStreamListFetcher(max_price_usd=max_price_usd, api_key=api_key, **fetcher_kwargs),
+            name=name,
+            description=description,
+            outputs_to_string={"source": "documents", "handler": _rows_as_text},
+        )

specstream_haystack-0.1.0.dist-info/METADATA

@@ -0,0 +1,83 @@
+Metadata-Version: 2.5
+Name: specstream-haystack
+Version: 0.1.0
+Summary: Give a Haystack pipeline or agent any list from the live web: ask in plain words, get every matching row as a Document with the page it came from, within a price you set in code.
+Project-URL: Homepage, https://specstream.ai
+Project-URL: Documentation, https://specstream.ai/docs
+Project-URL: Repository, https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack
+Project-URL: Issues, https://github.com/Specstreamai/specstream-tools/issues
+Project-URL: Pricing, https://specstream.ai/pricing
+Author-email: SpecStream <support@specstream.ai>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,haystack,haystack-ai,lead-lists,live-web,research,specstream,tools,web-data
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: docstring-parser
+Requires-Dist: haystack-ai>=2.14.0
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == 'test'
+Requires-Dist: ruff; extra == 'test'
+Description-Content-Type: text/markdown
+
+# specstream-haystack
+
+Give your Haystack pipeline or agent any list from the live web. Search components hand a pipeline pages to read; SpecStream hands it the finished list. Ask in plain words – "dental clinics in Lisbon with online booking", "US startups that filed a $5M+ Form D this week" – and get back every matching row as a `Document`, each with the page it was read from. The list starts only within a price you set in code, and the model can't raise it.
+
+```bash
+pip install -U specstream-haystack
+export SPECSTREAM_API_KEY=ss_live_...   # Agents & API in your SpecStream workspace
+```
+
+A free account comes with $10 of work to use within 30 days and needs no card: [specstream.ai](https://specstream.ai/signup).
+
+## In a pipeline
+
+```python
+from haystack_integrations.components.fetchers.specstream import SpecStreamListFetcher
+
+fetcher = SpecStreamListFetcher(max_price_usd=3, max_results=25)
+result = fetcher.run(request="Breweries in Portland, Oregon that host events, with their website")
+
+for doc in result["documents"]:
+    print(doc.meta["source_url"], doc.content)
+```
+
+Each `Document` is one row. `content` is the row as JSON. `meta` carries `source_url` (the page the row was read from), `checked_at`, `list_id`, `list_link`, `position`, `request` and `fields` (the row's columns as a dict).
+
+`run` also returns `list_id`, `status`, `list_link` and a `message` when the list was still running. When the quoted price is above `max_price_usd`, or the request is too vague to price, nothing runs and `SpecStreamError` explains why.
+
+## As an agent tool
+
+```python
+from haystack.components.agents import Agent
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.dataclasses import ChatMessage
+from haystack_integrations.tools.specstream import SpecStreamListTool
+
+agent = Agent(
+    chat_generator=OpenAIChatGenerator(model="gpt-4o"),
+    tools=[SpecStreamListTool(max_price_usd=5)],
+)
+result = agent.run(messages=[ChatMessage.from_user("Find open grants for urban beekeeping in the EU and list their deadlines.")])
+print(result["last_message"].text)
+```
+
+The agent writes the request and picks how many rows. You decide what one list may cost.
+
+## Retries don't buy twice
+
+The same request with the same limits, in the same hour, reaches the run it already started instead of starting another. A retried step never pays for the list twice.
+
+MIT licensed. Source: [Specstreamai/specstream-tools](https://github.com/Specstreamai/specstream-tools/tree/main/packages/specstream-haystack).

specstream_haystack-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+haystack_integrations/components/fetchers/specstream/__init__.py,sha256=-XhGjGz3lfhBzes6wbIxRr9f0QQJDkT43loVIJ9MNRc,242
+haystack_integrations/components/fetchers/specstream/_api.py,sha256=jzt7Ol2JPhgjUrQPMiiN9zR0nEXuMjVAt5oOS5pHqig,9511
+haystack_integrations/components/fetchers/specstream/list_fetcher.py,sha256=m325sq9oNcOvhNCkJotkmyQMfSbKnqQpJRLw_4VN6ZA,5274
+haystack_integrations/components/fetchers/specstream/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+haystack_integrations/tools/specstream/__init__.py,sha256=Iud2Umtey-0ipE-fX_oK7ycyusoYSaYbFBFyuNjdNYk,114
+haystack_integrations/tools/specstream/list_tool.py,sha256=QvwAeuOpm8GNmIySyuRgPOP31YipeGEtSClVKpdl0XM,2065
+specstream_haystack-0.1.0.dist-info/METADATA,sha256=hvRz5iOWWWLt5BaklsnTzlBHCzjRBrpEp12eO1q9UaU,4134
+specstream_haystack-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+specstream_haystack-0.1.0.dist-info/licenses/LICENSE,sha256=lvzGtfzsxc8Xe1_0sabdiWTcSDiSg3jqaFKV9x3F4oo,1071
+specstream_haystack-0.1.0.dist-info/RECORD,,

specstream_haystack-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

specstream_haystack-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 SpecStream SAS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.