proxmint-haystack 0.1.0__py3-none-any.whl

haystack_integrations/components/fetchers/proxmint/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: MIT
+from haystack_integrations.components.fetchers.proxmint._gateway import GatewayError, proxy_username
+from haystack_integrations.components.fetchers.proxmint.fetcher import ProxmintFetcher
+
+__all__ = ["GatewayError", "ProxmintFetcher", "proxy_username"]

haystack_integrations/components/fetchers/proxmint/_gateway.py

@@ -0,0 +1,239 @@
+"""Proxmint credentials, gateway usernames and one fetch through the proxy.
+
+Targeting rides in the proxy username: the login, then ``__``, then ``key.value``
+parameters joined by ``;``, for example ``login__cr.us;city.newyork;sessid.ab12;sessttl.30``.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import unicodedata
+from dataclasses import dataclass
+from html.parser import HTMLParser
+
+import httpx
+
+ACCOUNT_URL = "https://proxmint.com/api/v1/account"
+PRODUCTS = ("residential", "premium-residential", "datacenter", "mobile")
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
+)
+
+# What the gateway puts in the status line when it refuses a request itself.
+GATEWAY_ERRORS = {
+    "SITE_PERMANENTLY_BLOCKED": "The site is on the network's blocked list: https://proxmint.com/docs/blocked-websites",
+    "PORT_BLOCKED": "The destination port is closed. Web traffic works on 80, 443, 8080, 8443, 8888 and 8090.",
+    "HOST_BLOCKED": "You blocked this host yourself under Blocked hosts on your product page.",
+    "NO_USER": "The gateway doesn't recognise the login. Copy the credentials again from your dashboard.",
+    "TRAFFIC_EXHAUSTED": "The product's bandwidth is used up. Add a pack at https://proxmint.com/pricing",
+    "THREADS_EXHAUSTED": "Too many connections at once. Lower your concurrency.",
+    "USER_BLOCKED": "The proxy user is blocked, usually after many wrong passwords. Contact support@proxmint.com",
+    "NO_HOST_CONNECTION": "The exit IP couldn't reach the site. Retry: a new request gets a new IP.",
+    "NO_RAY": "No IPs match your targeting. Drop the city, ZIP or ASN, or check the location spelling.",
+}
+
+_FOLD = str.maketrans(
+    {"ł": "l", "ø": "o", "æ": "ae", "œ": "oe", "ß": "ss", "đ": "d", "ð": "d", "þ": "th", "ı": "i", "ħ": "h"}
+)
+
+
+class GatewayError(Exception):
+    """The Proxmint gateway refused the request (bad login, no bandwidth, blocked site...)."""
+
+
+@dataclass(frozen=True)
+class Credentials:
+    username: str
+    password: str
+    host: str = "gw.proxmint.com"
+    port: int = 823
+
+
+def load_credentials(
+    api_key: str | None = None,
+    product: str = "residential",
+    *,
+    username: str | None = None,
+    password: str | None = None,
+    client: httpx.Client | None = None,
+) -> Credentials:
+    """Proxy login for one product.
+
+    A username and password (arguments, else ``PROXMINT_USERNAME`` / ``PROXMINT_PASSWORD``)
+    are used as they are. Otherwise the API key (argument, else ``PROXMINT_API_KEY``) reads
+    them from the account, which also picks the right product.
+    """
+    if product not in PRODUCTS:
+        raise ValueError(f"product must be one of {', '.join(PRODUCTS)}")
+    if not api_key:
+        username = username or os.getenv("PROXMINT_USERNAME")
+        password = password or os.getenv("PROXMINT_PASSWORD")
+        if username and password:
+            return Credentials(username, password)
+        api_key = os.getenv("PROXMINT_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "Set PROXMINT_API_KEY (create one under Settings at https://proxmint.com/dashboard) "
+            "or PROXMINT_USERNAME and PROXMINT_PASSWORD."
+        )
+
+    get = client.get if client else httpx.get
+    r = get(ACCOUNT_URL, headers={"Authorization": f"Bearer {api_key}"}, timeout=20)
+    if r.status_code == 401:
+        raise ValueError("Proxmint rejected the API key. Create a new one under Settings in the dashboard.")
+    r.raise_for_status()
+    for p in r.json()["products"]:
+        if p["product"] != product:
+            continue
+        if p["status"] != "active":
+            raise ValueError(f"Your {product} product is {p['status']}. Top it up at https://proxmint.com/dashboard")
+        px = p["proxy"]
+        return Credentials(px["username"], px["password"], px["host"], int(px["httpPort"]))
+    raise ValueError(f"This account has no {product} product. Buy a pack at https://proxmint.com/pricing")
+
+
+def _geo(value: str) -> str:
+    """A place name in the gateway's form: "São Paulo" -> saopaulo, "Łódź" -> lodz."""
+    s = unicodedata.normalize("NFKD", value.lower())
+    s = "".join(c for c in s if not unicodedata.combining(c)).translate(_FOLD)
+    return re.sub(r"[^a-z0-9]", "", s)
+
+
+def proxy_username(
+    login: str,
+    *,
+    country: str | None = None,
+    state: str | None = None,
+    city: str | None = None,
+    zip_code: str | None = None,
+    asn: str | int | None = None,
+    session_id: str | None = None,
+    session_minutes: int | None = None,
+) -> str:
+    """The login with targeting and sticky-session parameters appended."""
+    if (state or city or zip_code or asn) and not country:
+        raise ValueError("state, city, zip_code and asn need a country as well.")
+    params = []
+    if country:
+        params.append("cr." + re.sub(r"[^a-z]", "", country.lower())[:2])
+    if city:
+        params.append("city." + _geo(city))
+    if state:
+        params.append("state." + _geo(state))
+    if zip_code:
+        params.append("zip." + str(zip_code).strip())
+    if asn:
+        params.append("asn." + re.sub(r"\D", "", str(asn)))
+    if session_id:
+        params.append("sessid." + _geo(session_id))
+    if session_minutes:
+        params.append(f"sessttl.{max(1, min(120, round(session_minutes)))}")
+    return f"{login}__{';'.join(params)}" if params else login
+
+
+def new_session_id() -> str:
+    return secrets.token_hex(6)
+
+
+def client_for(creds: Credentials, username: str, *, timeout: float = 60.0, headers: dict | None = None) -> httpx.Client:
+    """An HTTP client whose every request leaves through the gateway as ``username``."""
+    proxy = httpx.Proxy(f"http://{creds.host}:{creds.port}", auth=(username, creds.password))
+    return httpx.Client(
+        proxy=proxy,
+        timeout=timeout,
+        follow_redirects=True,
+        max_redirects=5,
+        headers={"User-Agent": USER_AGENT, **(headers or {})},
+    )
+
+
+@dataclass(frozen=True)
+class Page:
+    url: str
+    status: int
+    content_type: str
+    html: str
+    title: str | None
+
+    def text(self) -> str:
+        return html_to_text(self.html) if "html" in self.content_type else self.html
+
+
+def fetch(client: httpx.Client, url: str) -> Page:
+    """GET one page. Gateway refusals raise GatewayError; the site's own answers return as a Page."""
+    if not re.match(r"https?://", url):
+        raise ValueError(f"Not an http(s) URL: {url}")
+    try:
+        r = client.get(url)
+    except httpx.ProxyError as e:  # HTTPS: the refusal comes back on the CONNECT
+        raise _gateway_error(str(e)) from e
+    reason = r.reason_phrase.strip().upper()
+    if r.status_code in (400, 403, 407, 500, 502, 503) and reason in GATEWAY_ERRORS:  # plain HTTP
+        raise _gateway_error(f"{r.status_code} {reason}")
+    ctype = r.headers.get("content-type", "")
+    body = r.text
+    title = _Text.parse(body).title if "html" in ctype else None
+    return Page(str(r.url), r.status_code, ctype, body, title)
+
+
+def _gateway_error(status_line: str) -> GatewayError:
+    reason = status_line.split()[-1].upper() if status_line.split() else ""
+    hint = GATEWAY_ERRORS.get(reason, "See https://proxmint.com/docs/errors")
+    return GatewayError(f"Proxmint gateway: {status_line}. {hint}")
+
+
+class _Text(HTMLParser):
+    SKIP = {"script", "style", "noscript", "template", "svg", "head"}
+    BLOCK = {
+        "address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "figcaption", "footer",
+        "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre",
+        "section", "table", "td", "th", "tr", "ul",
+    }
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self.title: str | None = None
+        self._skip = 0
+        self._in_title = False
+
+    @classmethod
+    def parse(cls, html: str) -> _Text:
+        p = cls()
+        p.feed(html)
+        p.close()
+        return p
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "title":
+            self._in_title = self.title is None  # the page's own, not an <svg><title>
+        if tag == "body":
+            self._skip = 0  # </head> is optional in HTML
+        elif tag in self.SKIP:
+            self._skip += 1
+        elif tag in self.BLOCK:
+            self.parts.append("\n")
+
+    def handle_endtag(self, tag):
+        if tag == "title":
+            self._in_title = False
+        if tag in self.SKIP:
+            self._skip = max(0, self._skip - 1)
+        elif tag in self.BLOCK:
+            self.parts.append("\n")
+
+    def handle_data(self, data):
+        if self._in_title:
+            self.title = ((self.title or "") + data).strip()
+        elif not self._skip:
+            self.parts.append(data)
+
+
+def html_to_text(html: str) -> str:
+    """Readable text of a page: no scripts or styles, one line per block, blank lines squeezed."""
+    raw = "".join(_Text.parse(html).parts)
+    lines = (re.sub(r"[ \t\r\f\v]+", " ", line).strip() for line in raw.split("\n"))
+    return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()

haystack_integrations/components/fetchers/proxmint/fetcher.py

@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: MIT
+"""ProxmintFetcher: fetch web pages through Proxmint proxies and return Documents."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Literal
+
+import httpx
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+from haystack_integrations.components.fetchers.proxmint._gateway import (
+    Credentials,
+    GatewayError,
+    client_for,
+    fetch,
+    load_credentials,
+    new_session_id,
+    proxy_username,
+)
+
+logger = logging.getLogger(__name__)
+
+Product = Literal["residential", "premium-residential", "datacenter", "mobile"]
+
+
+@component
+class ProxmintFetcher:
+    """Fetch web pages through Proxmint residential, datacenter or mobile proxies.
+
+    Every URL leaves from a new exit IP, in the country, city or ASN you ask for. Set
+    ``session_minutes`` to keep one IP across runs. The output is plain Documents, so
+    it plugs into a splitter, a converter or an embedder.
+
+    ```python
+    from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+    fetcher = ProxmintFetcher(country="gb")  # reads PROXMINT_API_KEY
+    result = fetcher.run(urls=["https://example.com"])
+    print(result["documents"][0].content)
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("PROXMINT_API_KEY", strict=False),  # noqa: B008
+        *,
+        product: Product = "residential",
+        country: str | None = None,
+        state: str | None = None,
+        city: str | None = None,
+        zip_code: str | None = None,
+        asn: str | None = None,
+        session_minutes: int | None = None,
+        output: Literal["text", "html"] = "text",
+        timeout: float = 60.0,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        :param api_key: Proxmint API key (`pmk_...`, created under Settings in the dashboard). Without one,
+            `PROXMINT_USERNAME` and `PROXMINT_PASSWORD` are read instead.
+        :param product: `residential`, `premium-residential`, `datacenter` or `mobile`.
+        :param country: ISO 3166 alpha-2 code, for example `us`. State, city, ZIP and ASN need a country.
+        :param session_minutes: Keep one exit IP for this many minutes (1-120), across runs. Default: a new IP per URL.
+        :param output: `text` for readable text, `html` for the raw markup.
+        :param raise_on_failure: Raise on a failed URL instead of skipping it (it is listed in `failed_urls`).
+        """
+        self.api_key = api_key
+        self.product = product
+        self.targeting = {"country": country, "state": state, "city": city, "zip_code": zip_code, "asn": asn}
+        self.session_minutes = session_minutes
+        self.output = output
+        self.timeout = timeout
+        self.raise_on_failure = raise_on_failure
+        self._creds: Credentials | None = None
+        self._session_id = new_session_id() if session_minutes else None
+
+    def warm_up(self) -> None:
+        """Read the proxy login once, so a pipeline run makes no extra API call."""
+        if self._creds is None:
+            self._creds = load_credentials(self.api_key.resolve_value(), self.product)
+
+    @component.output_types(documents=list[Document], failed_urls=list[str])
+    def run(self, urls: list[str], country: str | None = None) -> dict[str, Any]:
+        """
+        :param urls: Pages to fetch (`http` or `https`).
+        :param country: Overrides the `country` set at init for this run (the state, city, ZIP and ASN set at init are dropped).
+        :returns: `documents` (content = page text or HTML, meta = source, status, title...) and `failed_urls`.
+        """
+        self.warm_up()
+        assert self._creds is not None
+        targeting = self.targeting
+        if country and country.lower() != (targeting["country"] or "").lower():
+            targeting = {"country": country, "state": None, "city": None, "zip_code": None, "asn": None}
+        login = proxy_username(
+            self._creds.username,
+            **targeting,
+            session_id=self._session_id,
+            session_minutes=self.session_minutes,
+        )
+        documents: list[Document] = []
+        failed: list[str] = []
+        with client_for(self._creds, login, timeout=self.timeout) as client:
+            for url in urls:
+                try:
+                    page = fetch(client, url)
+                except (GatewayError, httpx.HTTPError, ValueError) as e:
+                    if self.raise_on_failure:
+                        raise
+                    logger.warning("Skipped %s: %s", url, e)
+                    failed.append(url)
+                    continue
+                documents.append(
+                    Document(
+                        content=page.html if self.output == "html" else page.text(),
+                        meta={
+                            "source": page.url,
+                            "requested_url": url,
+                            "status": page.status,
+                            "title": page.title,
+                            "content_type": page.content_type,
+                            "country": targeting["country"],
+                        },
+                    )
+                )
+        return {"documents": documents, "failed_urls": failed}
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(
+            self,
+            api_key=self.api_key.to_dict(),
+            product=self.product,
+            session_minutes=self.session_minutes,
+            output=self.output,
+            timeout=self.timeout,
+            raise_on_failure=self.raise_on_failure,
+            **self.targeting,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProxmintFetcher:
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)

proxmint_haystack-0.1.0.dist-info/METADATA

@@ -0,0 +1,136 @@
+Metadata-Version: 2.5
+Name: proxmint-haystack
+Version: 0.1.0
+Summary: Haystack component that fetches web pages through Proxmint residential, datacenter and mobile proxies, with country, city and ASN targeting and sticky sessions.
+Project-URL: Homepage, https://proxmint.com
+Project-URL: Documentation, https://proxmint.com/docs/haystack
+Project-URL: Repository, https://github.com/proxmint/proxmint-haystack
+Project-URL: Issues, https://github.com/proxmint/proxmint-haystack/issues
+Author-email: Proxmint <support@proxmint.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,fetcher,haystack,proxmint,proxy,residential-proxy,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: haystack-ai>=2.10
+Requires-Dist: httpx<1,>=0.27
+Description-Content-Type: text/markdown
+
+# proxmint-haystack
+
+[Haystack](https://haystack.deepset.ai) integration for [Proxmint](https://proxmint.com) proxies: fetch web pages
+from residential, datacenter or 4G/5G mobile IPs and get them back as `Document`s. Pick the exit country, state,
+city, ZIP or ASN, and keep one IP for up to two hours when a site needs it.
+
+- **`ProxmintFetcher`**: a component that takes a list of URLs and returns `documents` (readable page text, or the
+  raw HTML) and `failed_urls`. Use it as the first step of an indexing pipeline, or wrap it with `ComponentTool` to
+  give an agent a "read this page from Germany" tool.
+
+Requests go straight from your machine to the Proxmint gateway (`gw.proxmint.com`). Nothing else sits in
+between, and bandwidth comes out of your own Proxmint account.
+
+## Install
+
+```bash
+pip install proxmint-haystack
+```
+
+## Credentials
+
+Create an API key under **Settings** in the [Proxmint dashboard](https://proxmint.com/dashboard) and export it:
+
+```bash
+export PROXMINT_API_KEY="pmk_..."
+```
+
+The key reads the proxy login for the product you choose (`residential` by default). You can use the proxy login
+instead, with `PROXMINT_USERNAME` and `PROXMINT_PASSWORD`.
+
+## Fetch pages
+
+```python
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+fetcher = ProxmintFetcher(country="gb")
+result = fetcher.run(urls=["https://example.com", "https://example.org"])
+
+print(result["documents"][0].content)
+print(result["documents"][0].meta)  # source, requested_url, status, title, content_type, country
+print(result["failed_urls"])
+```
+
+## Use it in a pipeline
+
+```python
+from haystack import Pipeline
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+store = InMemoryDocumentStore()
+pipe = Pipeline()
+pipe.add_component("fetcher", ProxmintFetcher(country="us", city="New York", session_minutes=30))
+pipe.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=5))
+pipe.add_component("writer", DocumentWriter(store))
+pipe.connect("fetcher.documents", "splitter.documents")
+pipe.connect("splitter.documents", "writer.documents")
+
+pipe.run({"fetcher": {"urls": ["https://example.com/pricing", "https://example.org/pricing"]}})
+```
+
+## Use it as an agent tool
+
+```python
+from haystack.tools import ComponentTool
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+tool = ComponentTool(
+    component=ProxmintFetcher(),
+    name="fetch_pages",
+    description="Fetch web pages through a proxy in a chosen country and return their text.",
+)
+```
+
+Pass the tool to an `Agent` or a chat generator as usual. The model can set `urls` and an optional `country` per call.
+
+## Options
+
+| Option | Default | What it does |
+| --- | --- | --- |
+| `api_key` | `PROXMINT_API_KEY` | A Haystack `Secret` |
+| `product` | `"residential"` | `residential`, `premium-residential`, `datacenter` or `mobile` |
+| `country` | any | Exit country (two-letter code). `run(urls, country=...)` overrides it for one run |
+| `state`, `city`, `zip_code`, `asn` | none | Finer targeting; needs `country` |
+| `session_minutes` | none | Keep one exit IP across runs for 1 to 120 minutes. Default: a new IP for every URL |
+| `output` | `"text"` | `"text"` for readable text, `"html"` for the raw page |
+| `timeout` | `60` | Seconds per request |
+| `raise_on_failure` | `False` | Raise on a failed URL instead of listing it in `failed_urls` |
+
+A gateway refusal (no bandwidth left, no IP matching the targeting, a blocked site) names what to change.
+
+## Good to know
+
+- Web traffic works on destination ports 80, 443, 8080, 8443, 8888 and 8090 ([port access](https://proxmint.com/docs/ports)).
+- A short list of sites, mostly banking and government, is refused by the network ([blocked websites](https://proxmint.com/docs/blocked-websites)).
+- City, state, ZIP and ASN targeting meter at 2x on residential, datacenter and mobile ([targeting](https://proxmint.com/docs/targeting)).
+- Each product allows up to 2,000 concurrent connections.
+
+Full guide: [proxmint.com/docs/haystack](https://proxmint.com/docs/haystack). Pricing: [proxmint.com/pricing](https://proxmint.com/pricing).
+Bandwidth is billed per GB and never expires.
+
+## Development
+
+```bash
+uv run --group test pytest
+```
+
+The tests run offline against a stand-in for the gateway.
+
+## License
+
+MIT

proxmint_haystack-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+haystack_integrations/components/fetchers/proxmint/__init__.py,sha256=Fupm6pf66-5PPYGjg0GqqRRUtaULm1dU6FYGSaZF6AI,284
+haystack_integrations/components/fetchers/proxmint/_gateway.py,sha256=l3HdW1glpWTG0OPPeyxOx3xOy5Ar4S1UkSWgi4BMfJ4,9163
+haystack_integrations/components/fetchers/proxmint/fetcher.py,sha256=cKW8Yn9AOraXOJIoXEaJ17_AMyTc_DgUOjKSKp38_Qo,5923
+proxmint_haystack-0.1.0.dist-info/METADATA,sha256=IleC4vTqn6GtKgru_yP0hC23rLrR3o_CEYPQm8_6SX4,5557
+proxmint_haystack-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+proxmint_haystack-0.1.0.dist-info/licenses/LICENSE,sha256=dPrcCX38Tr8hwRb1TsW0mdf20fN71cLQ_g6Y5MObf5w,1065
+proxmint_haystack-0.1.0.dist-info/RECORD,,

proxmint_haystack-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

proxmint_haystack-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Proxmint
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.