proxmint-haystack 0.1.0__tar.gz

proxmint_haystack-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,17 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ["3.10", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv run --python ${{ matrix.python }} --group test pytest -q

proxmint_haystack-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,24 @@
+# Publishes to PyPI when a version tag (v0.1.0, ...) is pushed. PyPI trusted publishing:
+# no token is stored anywhere. One-time setup on pypi.org, under "Publishing" in the
+# account: add a pending publisher with project proxmint-haystack, owner proxmint,
+# repository proxmint-haystack, workflow publish.yml, environment pypi.
+
+name: Publish
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v6
+      - run: uv run --group test pytest -q
+      - run: uv build
+      - run: uv publish

proxmint_haystack-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.pyc
+.venv/
+dist/
+.pytest_cache/
+uv.lock
+.benchmarks/

proxmint_haystack-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Proxmint
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

proxmint_haystack-0.1.0/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.5
+Name: proxmint-haystack
+Version: 0.1.0
+Summary: Haystack component that fetches web pages through Proxmint residential, datacenter and mobile proxies, with country, city and ASN targeting and sticky sessions.
+Project-URL: Homepage, https://proxmint.com
+Project-URL: Documentation, https://proxmint.com/docs/haystack
+Project-URL: Repository, https://github.com/proxmint/proxmint-haystack
+Project-URL: Issues, https://github.com/proxmint/proxmint-haystack/issues
+Author-email: Proxmint <support@proxmint.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agents,fetcher,haystack,proxmint,proxy,residential-proxy,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: Proxy Servers
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: haystack-ai>=2.10
+Requires-Dist: httpx<1,>=0.27
+Description-Content-Type: text/markdown
+
+# proxmint-haystack
+
+[Haystack](https://haystack.deepset.ai) integration for [Proxmint](https://proxmint.com) proxies: fetch web pages
+from residential, datacenter or 4G/5G mobile IPs and get them back as `Document`s. Pick the exit country, state,
+city, ZIP or ASN, and keep one IP for up to two hours when a site needs it.
+
+- **`ProxmintFetcher`**: a component that takes a list of URLs and returns `documents` (readable page text, or the
+  raw HTML) and `failed_urls`. Use it as the first step of an indexing pipeline, or wrap it with `ComponentTool` to
+  give an agent a "read this page from Germany" tool.
+
+Requests go straight from your machine to the Proxmint gateway (`gw.proxmint.com`). Nothing else sits in
+between, and bandwidth comes out of your own Proxmint account.
+
+## Install
+
+```bash
+pip install proxmint-haystack
+```
+
+## Credentials
+
+Create an API key under **Settings** in the [Proxmint dashboard](https://proxmint.com/dashboard) and export it:
+
+```bash
+export PROXMINT_API_KEY="pmk_..."
+```
+
+The key reads the proxy login for the product you choose (`residential` by default). You can use the proxy login
+instead, with `PROXMINT_USERNAME` and `PROXMINT_PASSWORD`.
+
+## Fetch pages
+
+```python
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+fetcher = ProxmintFetcher(country="gb")
+result = fetcher.run(urls=["https://example.com", "https://example.org"])
+
+print(result["documents"][0].content)
+print(result["documents"][0].meta)  # source, requested_url, status, title, content_type, country
+print(result["failed_urls"])
+```
+
+## Use it in a pipeline
+
+```python
+from haystack import Pipeline
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+store = InMemoryDocumentStore()
+pipe = Pipeline()
+pipe.add_component("fetcher", ProxmintFetcher(country="us", city="New York", session_minutes=30))
+pipe.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=5))
+pipe.add_component("writer", DocumentWriter(store))
+pipe.connect("fetcher.documents", "splitter.documents")
+pipe.connect("splitter.documents", "writer.documents")
+
+pipe.run({"fetcher": {"urls": ["https://example.com/pricing", "https://example.org/pricing"]}})
+```
+
+## Use it as an agent tool
+
+```python
+from haystack.tools import ComponentTool
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+tool = ComponentTool(
+    component=ProxmintFetcher(),
+    name="fetch_pages",
+    description="Fetch web pages through a proxy in a chosen country and return their text.",
+)
+```
+
+Pass the tool to an `Agent` or a chat generator as usual. The model can set `urls` and an optional `country` per call.
+
+## Options
+
+| Option | Default | What it does |
+| --- | --- | --- |
+| `api_key` | `PROXMINT_API_KEY` | A Haystack `Secret` |
+| `product` | `"residential"` | `residential`, `premium-residential`, `datacenter` or `mobile` |
+| `country` | any | Exit country (two-letter code). `run(urls, country=...)` overrides it for one run |
+| `state`, `city`, `zip_code`, `asn` | none | Finer targeting; needs `country` |
+| `session_minutes` | none | Keep one exit IP across runs for 1 to 120 minutes. Default: a new IP for every URL |
+| `output` | `"text"` | `"text"` for readable text, `"html"` for the raw page |
+| `timeout` | `60` | Seconds per request |
+| `raise_on_failure` | `False` | Raise on a failed URL instead of listing it in `failed_urls` |
+
+A gateway refusal (no bandwidth left, no IP matching the targeting, a blocked site) names what to change.
+
+## Good to know
+
+- Web traffic works on destination ports 80, 443, 8080, 8443, 8888 and 8090 ([port access](https://proxmint.com/docs/ports)).
+- A short list of sites, mostly banking and government, is refused by the network ([blocked websites](https://proxmint.com/docs/blocked-websites)).
+- City, state, ZIP and ASN targeting meter at 2x on residential, datacenter and mobile ([targeting](https://proxmint.com/docs/targeting)).
+- Each product allows up to 2,000 concurrent connections.
+
+Full guide: [proxmint.com/docs/haystack](https://proxmint.com/docs/haystack). Pricing: [proxmint.com/pricing](https://proxmint.com/pricing).
+Bandwidth is billed per GB and never expires.
+
+## Development
+
+```bash
+uv run --group test pytest
+```
+
+The tests run offline against a stand-in for the gateway.
+
+## License
+
+MIT

proxmint_haystack-0.1.0/README.md

@@ -0,0 +1,114 @@
+# proxmint-haystack
+
+[Haystack](https://haystack.deepset.ai) integration for [Proxmint](https://proxmint.com) proxies: fetch web pages
+from residential, datacenter or 4G/5G mobile IPs and get them back as `Document`s. Pick the exit country, state,
+city, ZIP or ASN, and keep one IP for up to two hours when a site needs it.
+
+- **`ProxmintFetcher`**: a component that takes a list of URLs and returns `documents` (readable page text, or the
+  raw HTML) and `failed_urls`. Use it as the first step of an indexing pipeline, or wrap it with `ComponentTool` to
+  give an agent a "read this page from Germany" tool.
+
+Requests go straight from your machine to the Proxmint gateway (`gw.proxmint.com`). Nothing else sits in
+between, and bandwidth comes out of your own Proxmint account.
+
+## Install
+
+```bash
+pip install proxmint-haystack
+```
+
+## Credentials
+
+Create an API key under **Settings** in the [Proxmint dashboard](https://proxmint.com/dashboard) and export it:
+
+```bash
+export PROXMINT_API_KEY="pmk_..."
+```
+
+The key reads the proxy login for the product you choose (`residential` by default). You can use the proxy login
+instead, with `PROXMINT_USERNAME` and `PROXMINT_PASSWORD`.
+
+## Fetch pages
+
+```python
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+fetcher = ProxmintFetcher(country="gb")
+result = fetcher.run(urls=["https://example.com", "https://example.org"])
+
+print(result["documents"][0].content)
+print(result["documents"][0].meta)  # source, requested_url, status, title, content_type, country
+print(result["failed_urls"])
+```
+
+## Use it in a pipeline
+
+```python
+from haystack import Pipeline
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+store = InMemoryDocumentStore()
+pipe = Pipeline()
+pipe.add_component("fetcher", ProxmintFetcher(country="us", city="New York", session_minutes=30))
+pipe.add_component("splitter", DocumentSplitter(split_by="sentence", split_length=5))
+pipe.add_component("writer", DocumentWriter(store))
+pipe.connect("fetcher.documents", "splitter.documents")
+pipe.connect("splitter.documents", "writer.documents")
+
+pipe.run({"fetcher": {"urls": ["https://example.com/pricing", "https://example.org/pricing"]}})
+```
+
+## Use it as an agent tool
+
+```python
+from haystack.tools import ComponentTool
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+tool = ComponentTool(
+    component=ProxmintFetcher(),
+    name="fetch_pages",
+    description="Fetch web pages through a proxy in a chosen country and return their text.",
+)
+```
+
+Pass the tool to an `Agent` or a chat generator as usual. The model can set `urls` and an optional `country` per call.
+
+## Options
+
+| Option | Default | What it does |
+| --- | --- | --- |
+| `api_key` | `PROXMINT_API_KEY` | A Haystack `Secret` |
+| `product` | `"residential"` | `residential`, `premium-residential`, `datacenter` or `mobile` |
+| `country` | any | Exit country (two-letter code). `run(urls, country=...)` overrides it for one run |
+| `state`, `city`, `zip_code`, `asn` | none | Finer targeting; needs `country` |
+| `session_minutes` | none | Keep one exit IP across runs for 1 to 120 minutes. Default: a new IP for every URL |
+| `output` | `"text"` | `"text"` for readable text, `"html"` for the raw page |
+| `timeout` | `60` | Seconds per request |
+| `raise_on_failure` | `False` | Raise on a failed URL instead of listing it in `failed_urls` |
+
+A gateway refusal (no bandwidth left, no IP matching the targeting, a blocked site) names what to change.
+
+## Good to know
+
+- Web traffic works on destination ports 80, 443, 8080, 8443, 8888 and 8090 ([port access](https://proxmint.com/docs/ports)).
+- A short list of sites, mostly banking and government, is refused by the network ([blocked websites](https://proxmint.com/docs/blocked-websites)).
+- City, state, ZIP and ASN targeting meter at 2x on residential, datacenter and mobile ([targeting](https://proxmint.com/docs/targeting)).
+- Each product allows up to 2,000 concurrent connections.
+
+Full guide: [proxmint.com/docs/haystack](https://proxmint.com/docs/haystack). Pricing: [proxmint.com/pricing](https://proxmint.com/pricing).
+Bandwidth is billed per GB and never expires.
+
+## Development
+
+```bash
+uv run --group test pytest
+```
+
+The tests run offline against a stand-in for the gateway.
+
+## License
+
+MIT

proxmint_haystack-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "proxmint-haystack"
+version = "0.1.0"
+description = "Haystack component that fetches web pages through Proxmint residential, datacenter and mobile proxies, with country, city and ASN targeting and sticky sessions."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.10"
+authors = [{ name = "Proxmint", email = "support@proxmint.com" }]
+keywords = ["haystack", "proxy", "residential-proxy", "web-scraping", "fetcher", "agents", "proxmint"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Topic :: Internet :: Proxy Servers",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = ["haystack-ai>=2.10", "httpx>=0.27,<1"]
+
+[project.urls]
+Homepage = "https://proxmint.com"
+Documentation = "https://proxmint.com/docs/haystack"
+Repository = "https://github.com/proxmint/proxmint-haystack"
+Issues = "https://github.com/proxmint/proxmint-haystack/issues"
+
+[dependency-groups]
+test = ["pytest>=8"]
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/haystack_integrations"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

proxmint_haystack-0.1.0/src/haystack_integrations/components/fetchers/proxmint/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: MIT
+from haystack_integrations.components.fetchers.proxmint._gateway import GatewayError, proxy_username
+from haystack_integrations.components.fetchers.proxmint.fetcher import ProxmintFetcher
+
+__all__ = ["GatewayError", "ProxmintFetcher", "proxy_username"]

proxmint_haystack-0.1.0/src/haystack_integrations/components/fetchers/proxmint/_gateway.py

@@ -0,0 +1,239 @@
+"""Proxmint credentials, gateway usernames and one fetch through the proxy.
+
+Targeting rides in the proxy username: the login, then ``__``, then ``key.value``
+parameters joined by ``;``, for example ``login__cr.us;city.newyork;sessid.ab12;sessttl.30``.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import unicodedata
+from dataclasses import dataclass
+from html.parser import HTMLParser
+
+import httpx
+
+ACCOUNT_URL = "https://proxmint.com/api/v1/account"
+PRODUCTS = ("residential", "premium-residential", "datacenter", "mobile")
+USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"
+)
+
+# What the gateway puts in the status line when it refuses a request itself.
+GATEWAY_ERRORS = {
+    "SITE_PERMANENTLY_BLOCKED": "The site is on the network's blocked list: https://proxmint.com/docs/blocked-websites",
+    "PORT_BLOCKED": "The destination port is closed. Web traffic works on 80, 443, 8080, 8443, 8888 and 8090.",
+    "HOST_BLOCKED": "You blocked this host yourself under Blocked hosts on your product page.",
+    "NO_USER": "The gateway doesn't recognise the login. Copy the credentials again from your dashboard.",
+    "TRAFFIC_EXHAUSTED": "The product's bandwidth is used up. Add a pack at https://proxmint.com/pricing",
+    "THREADS_EXHAUSTED": "Too many connections at once. Lower your concurrency.",
+    "USER_BLOCKED": "The proxy user is blocked, usually after many wrong passwords. Contact support@proxmint.com",
+    "NO_HOST_CONNECTION": "The exit IP couldn't reach the site. Retry: a new request gets a new IP.",
+    "NO_RAY": "No IPs match your targeting. Drop the city, ZIP or ASN, or check the location spelling.",
+}
+
+_FOLD = str.maketrans(
+    {"ł": "l", "ø": "o", "æ": "ae", "œ": "oe", "ß": "ss", "đ": "d", "ð": "d", "þ": "th", "ı": "i", "ħ": "h"}
+)
+
+
+class GatewayError(Exception):
+    """The Proxmint gateway refused the request (bad login, no bandwidth, blocked site...)."""
+
+
+@dataclass(frozen=True)
+class Credentials:
+    username: str
+    password: str
+    host: str = "gw.proxmint.com"
+    port: int = 823
+
+
+def load_credentials(
+    api_key: str | None = None,
+    product: str = "residential",
+    *,
+    username: str | None = None,
+    password: str | None = None,
+    client: httpx.Client | None = None,
+) -> Credentials:
+    """Proxy login for one product.
+
+    A username and password (arguments, else ``PROXMINT_USERNAME`` / ``PROXMINT_PASSWORD``)
+    are used as they are. Otherwise the API key (argument, else ``PROXMINT_API_KEY``) reads
+    them from the account, which also picks the right product.
+    """
+    if product not in PRODUCTS:
+        raise ValueError(f"product must be one of {', '.join(PRODUCTS)}")
+    if not api_key:
+        username = username or os.getenv("PROXMINT_USERNAME")
+        password = password or os.getenv("PROXMINT_PASSWORD")
+        if username and password:
+            return Credentials(username, password)
+        api_key = os.getenv("PROXMINT_API_KEY")
+    if not api_key:
+        raise ValueError(
+            "Set PROXMINT_API_KEY (create one under Settings at https://proxmint.com/dashboard) "
+            "or PROXMINT_USERNAME and PROXMINT_PASSWORD."
+        )
+
+    get = client.get if client else httpx.get
+    r = get(ACCOUNT_URL, headers={"Authorization": f"Bearer {api_key}"}, timeout=20)
+    if r.status_code == 401:
+        raise ValueError("Proxmint rejected the API key. Create a new one under Settings in the dashboard.")
+    r.raise_for_status()
+    for p in r.json()["products"]:
+        if p["product"] != product:
+            continue
+        if p["status"] != "active":
+            raise ValueError(f"Your {product} product is {p['status']}. Top it up at https://proxmint.com/dashboard")
+        px = p["proxy"]
+        return Credentials(px["username"], px["password"], px["host"], int(px["httpPort"]))
+    raise ValueError(f"This account has no {product} product. Buy a pack at https://proxmint.com/pricing")
+
+
+def _geo(value: str) -> str:
+    """A place name in the gateway's form: "São Paulo" -> saopaulo, "Łódź" -> lodz."""
+    s = unicodedata.normalize("NFKD", value.lower())
+    s = "".join(c for c in s if not unicodedata.combining(c)).translate(_FOLD)
+    return re.sub(r"[^a-z0-9]", "", s)
+
+
+def proxy_username(
+    login: str,
+    *,
+    country: str | None = None,
+    state: str | None = None,
+    city: str | None = None,
+    zip_code: str | None = None,
+    asn: str | int | None = None,
+    session_id: str | None = None,
+    session_minutes: int | None = None,
+) -> str:
+    """The login with targeting and sticky-session parameters appended."""
+    if (state or city or zip_code or asn) and not country:
+        raise ValueError("state, city, zip_code and asn need a country as well.")
+    params = []
+    if country:
+        params.append("cr." + re.sub(r"[^a-z]", "", country.lower())[:2])
+    if city:
+        params.append("city." + _geo(city))
+    if state:
+        params.append("state." + _geo(state))
+    if zip_code:
+        params.append("zip." + str(zip_code).strip())
+    if asn:
+        params.append("asn." + re.sub(r"\D", "", str(asn)))
+    if session_id:
+        params.append("sessid." + _geo(session_id))
+    if session_minutes:
+        params.append(f"sessttl.{max(1, min(120, round(session_minutes)))}")
+    return f"{login}__{';'.join(params)}" if params else login
+
+
+def new_session_id() -> str:
+    return secrets.token_hex(6)
+
+
+def client_for(creds: Credentials, username: str, *, timeout: float = 60.0, headers: dict | None = None) -> httpx.Client:
+    """An HTTP client whose every request leaves through the gateway as ``username``."""
+    proxy = httpx.Proxy(f"http://{creds.host}:{creds.port}", auth=(username, creds.password))
+    return httpx.Client(
+        proxy=proxy,
+        timeout=timeout,
+        follow_redirects=True,
+        max_redirects=5,
+        headers={"User-Agent": USER_AGENT, **(headers or {})},
+    )
+
+
+@dataclass(frozen=True)
+class Page:
+    url: str
+    status: int
+    content_type: str
+    html: str
+    title: str | None
+
+    def text(self) -> str:
+        return html_to_text(self.html) if "html" in self.content_type else self.html
+
+
+def fetch(client: httpx.Client, url: str) -> Page:
+    """GET one page. Gateway refusals raise GatewayError; the site's own answers return as a Page."""
+    if not re.match(r"https?://", url):
+        raise ValueError(f"Not an http(s) URL: {url}")
+    try:
+        r = client.get(url)
+    except httpx.ProxyError as e:  # HTTPS: the refusal comes back on the CONNECT
+        raise _gateway_error(str(e)) from e
+    reason = r.reason_phrase.strip().upper()
+    if r.status_code in (400, 403, 407, 500, 502, 503) and reason in GATEWAY_ERRORS:  # plain HTTP
+        raise _gateway_error(f"{r.status_code} {reason}")
+    ctype = r.headers.get("content-type", "")
+    body = r.text
+    title = _Text.parse(body).title if "html" in ctype else None
+    return Page(str(r.url), r.status_code, ctype, body, title)
+
+
+def _gateway_error(status_line: str) -> GatewayError:
+    reason = status_line.split()[-1].upper() if status_line.split() else ""
+    hint = GATEWAY_ERRORS.get(reason, "See https://proxmint.com/docs/errors")
+    return GatewayError(f"Proxmint gateway: {status_line}. {hint}")
+
+
+class _Text(HTMLParser):
+    SKIP = {"script", "style", "noscript", "template", "svg", "head"}
+    BLOCK = {
+        "address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "figcaption", "footer",
+        "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "ol", "p", "pre",
+        "section", "table", "td", "th", "tr", "ul",
+    }
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self.title: str | None = None
+        self._skip = 0
+        self._in_title = False
+
+    @classmethod
+    def parse(cls, html: str) -> _Text:
+        p = cls()
+        p.feed(html)
+        p.close()
+        return p
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "title":
+            self._in_title = self.title is None  # the page's own, not an <svg><title>
+        if tag == "body":
+            self._skip = 0  # </head> is optional in HTML
+        elif tag in self.SKIP:
+            self._skip += 1
+        elif tag in self.BLOCK:
+            self.parts.append("\n")
+
+    def handle_endtag(self, tag):
+        if tag == "title":
+            self._in_title = False
+        if tag in self.SKIP:
+            self._skip = max(0, self._skip - 1)
+        elif tag in self.BLOCK:
+            self.parts.append("\n")
+
+    def handle_data(self, data):
+        if self._in_title:
+            self.title = ((self.title or "") + data).strip()
+        elif not self._skip:
+            self.parts.append(data)
+
+
+def html_to_text(html: str) -> str:
+    """Readable text of a page: no scripts or styles, one line per block, blank lines squeezed."""
+    raw = "".join(_Text.parse(html).parts)
+    lines = (re.sub(r"[ \t\r\f\v]+", " ", line).strip() for line in raw.split("\n"))
+    return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip()

proxmint_haystack-0.1.0/src/haystack_integrations/components/fetchers/proxmint/fetcher.py

@@ -0,0 +1,144 @@
+# SPDX-License-Identifier: MIT
+"""ProxmintFetcher: fetch web pages through Proxmint proxies and return Documents."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Literal
+
+import httpx
+from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+from haystack_integrations.components.fetchers.proxmint._gateway import (
+    Credentials,
+    GatewayError,
+    client_for,
+    fetch,
+    load_credentials,
+    new_session_id,
+    proxy_username,
+)
+
+logger = logging.getLogger(__name__)
+
+Product = Literal["residential", "premium-residential", "datacenter", "mobile"]
+
+
+@component
+class ProxmintFetcher:
+    """Fetch web pages through Proxmint residential, datacenter or mobile proxies.
+
+    Every URL leaves from a new exit IP, in the country, city or ASN you ask for. Set
+    ``session_minutes`` to keep one IP across runs. The output is plain Documents, so
+    it plugs into a splitter, a converter or an embedder.
+
+    ```python
+    from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+
+    fetcher = ProxmintFetcher(country="gb")  # reads PROXMINT_API_KEY
+    result = fetcher.run(urls=["https://example.com"])
+    print(result["documents"][0].content)
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("PROXMINT_API_KEY", strict=False),  # noqa: B008
+        *,
+        product: Product = "residential",
+        country: str | None = None,
+        state: str | None = None,
+        city: str | None = None,
+        zip_code: str | None = None,
+        asn: str | None = None,
+        session_minutes: int | None = None,
+        output: Literal["text", "html"] = "text",
+        timeout: float = 60.0,
+        raise_on_failure: bool = False,
+    ) -> None:
+        """
+        :param api_key: Proxmint API key (`pmk_...`, created under Settings in the dashboard). Without one,
+            `PROXMINT_USERNAME` and `PROXMINT_PASSWORD` are read instead.
+        :param product: `residential`, `premium-residential`, `datacenter` or `mobile`.
+        :param country: ISO 3166 alpha-2 code, for example `us`. State, city, ZIP and ASN need a country.
+        :param session_minutes: Keep one exit IP for this many minutes (1-120), across runs. Default: a new IP per URL.
+        :param output: `text` for readable text, `html` for the raw markup.
+        :param raise_on_failure: Raise on a failed URL instead of skipping it (it is listed in `failed_urls`).
+        """
+        self.api_key = api_key
+        self.product = product
+        self.targeting = {"country": country, "state": state, "city": city, "zip_code": zip_code, "asn": asn}
+        self.session_minutes = session_minutes
+        self.output = output
+        self.timeout = timeout
+        self.raise_on_failure = raise_on_failure
+        self._creds: Credentials | None = None
+        self._session_id = new_session_id() if session_minutes else None
+
+    def warm_up(self) -> None:
+        """Read the proxy login once, so a pipeline run makes no extra API call."""
+        if self._creds is None:
+            self._creds = load_credentials(self.api_key.resolve_value(), self.product)
+
+    @component.output_types(documents=list[Document], failed_urls=list[str])
+    def run(self, urls: list[str], country: str | None = None) -> dict[str, Any]:
+        """
+        :param urls: Pages to fetch (`http` or `https`).
+        :param country: Overrides the `country` set at init for this run (the state, city, ZIP and ASN set at init are dropped).
+        :returns: `documents` (content = page text or HTML, meta = source, status, title...) and `failed_urls`.
+        """
+        self.warm_up()
+        assert self._creds is not None
+        targeting = self.targeting
+        if country and country.lower() != (targeting["country"] or "").lower():
+            targeting = {"country": country, "state": None, "city": None, "zip_code": None, "asn": None}
+        login = proxy_username(
+            self._creds.username,
+            **targeting,
+            session_id=self._session_id,
+            session_minutes=self.session_minutes,
+        )
+        documents: list[Document] = []
+        failed: list[str] = []
+        with client_for(self._creds, login, timeout=self.timeout) as client:
+            for url in urls:
+                try:
+                    page = fetch(client, url)
+                except (GatewayError, httpx.HTTPError, ValueError) as e:
+                    if self.raise_on_failure:
+                        raise
+                    logger.warning("Skipped %s: %s", url, e)
+                    failed.append(url)
+                    continue
+                documents.append(
+                    Document(
+                        content=page.html if self.output == "html" else page.text(),
+                        meta={
+                            "source": page.url,
+                            "requested_url": url,
+                            "status": page.status,
+                            "title": page.title,
+                            "content_type": page.content_type,
+                            "country": targeting["country"],
+                        },
+                    )
+                )
+        return {"documents": documents, "failed_urls": failed}
+
+    def to_dict(self) -> dict[str, Any]:
+        return default_to_dict(
+            self,
+            api_key=self.api_key.to_dict(),
+            product=self.product,
+            session_minutes=self.session_minutes,
+            output=self.output,
+            timeout=self.timeout,
+            raise_on_failure=self.raise_on_failure,
+            **self.targeting,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProxmintFetcher:
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)

proxmint_haystack-0.1.0/tests/test_fetcher.py

@@ -0,0 +1,88 @@
+import httpx
+import pytest
+from haystack import Pipeline
+from haystack.utils import Secret
+
+from haystack_integrations.components.fetchers.proxmint import ProxmintFetcher
+from haystack_integrations.components.fetchers.proxmint._gateway import Credentials, GatewayError
+
+CREDS = Credentials("login", "secret")
+MODULE = "haystack_integrations.components.fetchers.proxmint.fetcher"
+
+
+@pytest.fixture
+def gateway(monkeypatch):
+    """Stands in for the gateway: records the proxy username and answers with a small page."""
+    seen: list[str] = []
+
+    def client_for(creds, username, *, timeout, headers=None):
+        seen.append(username)
+
+        def handler(req):
+            if "fail" in req.url.host:
+                return httpx.Response(407, extensions={"reason_phrase": b"TRAFFIC_EXHAUSTED"})
+            return httpx.Response(
+                200,
+                headers={"content-type": "text/html"},
+                text=f"<title>Page {req.url.host}</title><p>Hello from {req.url.host}</p>",
+            )
+
+        return httpx.Client(transport=httpx.MockTransport(handler))
+
+    monkeypatch.setattr(f"{MODULE}.client_for", client_for)
+    monkeypatch.setattr(f"{MODULE}.load_credentials", lambda *a, **k: CREDS)
+    return seen
+
+
+def test_documents_and_failed_urls(gateway):
+    out = ProxmintFetcher(country="gb").run(["https://a.example/", "http://fail.example/", "https://b.example/"])
+    assert [d.content for d in out["documents"]] == ["Hello from a.example", "Hello from b.example"]
+    assert out["documents"][0].meta["title"] == "Page a.example"
+    assert out["documents"][0].meta["country"] == "gb"
+    assert out["failed_urls"] == ["http://fail.example/"]
+    assert gateway == ["login__cr.gb"]
+
+
+def test_runtime_country_and_sticky_session(gateway):
+    fetcher = ProxmintFetcher(country="us", city="New York", session_minutes=10)
+    fetcher.run(["https://a.example/"])
+    fetcher.run(["https://a.example/"], country="de")
+    sid = gateway[0].split("sessid.")[1].split(";")[0]
+    assert gateway[0] == f"login__cr.us;city.newyork;sessid.{sid};sessttl.10"
+    assert gateway[1].startswith("login__cr.de;sessid.") and "city" not in gateway[1]
+
+
+def test_html_output(gateway):
+    out = ProxmintFetcher(output="html").run(["https://a.example/"])
+    assert out["documents"][0].content.startswith("<title>Page a.example</title>")
+
+
+def test_raise_on_failure(gateway):
+    with pytest.raises(GatewayError, match="bandwidth is used up"):
+        ProxmintFetcher(raise_on_failure=True).run(["http://fail.example/"])
+
+
+def test_serialization_round_trip(monkeypatch):
+    monkeypatch.setenv("PROXMINT_API_KEY", "pmk_test")
+    fetcher = ProxmintFetcher(product="mobile", country="br", city="São Paulo", session_minutes=5, output="html")
+    data = fetcher.to_dict()
+    assert data["type"].endswith("ProxmintFetcher")
+    assert data["init_parameters"]["api_key"] == {"env_vars": ["PROXMINT_API_KEY"], "strict": False, "type": "env_var"}
+    again = ProxmintFetcher.from_dict(data)
+    assert (again.product, again.targeting["city"], again.session_minutes, again.output) == ("mobile", "São Paulo", 5, "html")
+    assert isinstance(again.api_key, Secret)
+
+
+def test_in_a_pipeline(gateway):
+    pipe = Pipeline()
+    pipe.add_component("fetcher", ProxmintFetcher())
+    out = pipe.run({"fetcher": {"urls": ["https://a.example/"]}})
+    assert out["fetcher"]["documents"][0].content == "Hello from a.example"
+
+
+def test_as_agent_tool(gateway):
+    from haystack.tools import ComponentTool
+
+    tool = ComponentTool(component=ProxmintFetcher(), name="fetch_pages", description="Fetch pages")
+    assert set(tool.parameters["properties"]) == {"urls", "country"} and tool.parameters["required"] == ["urls"]
+    assert tool.function(urls=["https://a.example/"])["documents"][0].content == "Hello from a.example"

proxmint_haystack-0.1.0/tests/test_gateway.py

@@ -0,0 +1,101 @@
+import httpx
+import pytest
+
+from haystack_integrations.components.fetchers.proxmint._gateway import GatewayError, fetch, html_to_text, load_credentials, proxy_username
+
+ACCOUNT = {
+    "balance": {"cents": 0, "usd": "0.00"},
+    "products": [
+        {
+            "product": "residential",
+            "status": "active",
+            "proxy": {"host": "gw.proxmint.com", "httpPort": 823, "socks5Port": 824, "username": "u1", "password": "p1"},
+        },
+        {
+            "product": "mobile",
+            "status": "paused",
+            "proxy": {"host": "gw.proxmint.com", "httpPort": 823, "socks5Port": 824, "username": "u2", "password": "p2"},
+        },
+    ],
+}
+
+
+def account_client(status=200, body=ACCOUNT):
+    def handler(req: httpx.Request) -> httpx.Response:
+        assert req.headers["authorization"] == "Bearer pmk_test"
+        return httpx.Response(status, json=body)
+
+    return httpx.Client(transport=httpx.MockTransport(handler))
+
+
+def test_username_targeting():
+    assert proxy_username("u") == "u"
+    assert proxy_username("u", country="US") == "u__cr.us"
+    assert (
+        proxy_username("u", country="br", city="São Paulo", session_id="ab12", session_minutes=30)
+        == "u__cr.br;city.saopaulo;sessid.ab12;sessttl.30"
+    )
+    assert proxy_username("u", country="pl", city="Łódź") == "u__cr.pl;city.lodz"
+    assert proxy_username("u", country="us", state="New York", zip_code=" 10001 ", asn="AS7922") == (
+        "u__cr.us;state.newyork;zip.10001;asn.7922"
+    )
+    assert proxy_username("u", session_id="x", session_minutes=500).endswith("sessttl.120")
+
+
+def test_fine_geo_needs_a_country():
+    with pytest.raises(ValueError, match="need a country"):
+        proxy_username("u", city="paris")
+
+
+def test_credentials_from_env(monkeypatch):
+    monkeypatch.delenv("PROXMINT_API_KEY", raising=False)
+    monkeypatch.setenv("PROXMINT_USERNAME", "envuser")
+    monkeypatch.setenv("PROXMINT_PASSWORD", "envpass")
+    c = load_credentials()
+    assert (c.username, c.password, c.host, c.port) == ("envuser", "envpass", "gw.proxmint.com", 823)
+
+
+def test_credentials_from_api_key(monkeypatch):
+    monkeypatch.delenv("PROXMINT_USERNAME", raising=False)
+    c = load_credentials("pmk_test", client=account_client())
+    assert (c.username, c.password) == ("u1", "p1")
+    with pytest.raises(ValueError, match="mobile product is paused"):
+        load_credentials("pmk_test", "mobile", client=account_client())
+    with pytest.raises(ValueError, match="no datacenter product"):
+        load_credentials("pmk_test", "datacenter", client=account_client())
+    with pytest.raises(ValueError, match="rejected the API key"):
+        load_credentials("pmk_test", client=account_client(401, {"error": "invalid"}))
+
+
+def test_no_credentials(monkeypatch):
+    for k in ("PROXMINT_API_KEY", "PROXMINT_USERNAME", "PROXMINT_PASSWORD"):
+        monkeypatch.delenv(k, raising=False)
+    with pytest.raises(ValueError, match="PROXMINT_API_KEY"):
+        load_credentials()
+
+
+def page_client(status=200, reason=b"OK", html="<html><head><title>Hi</title></head><body><p>x</p></body></html>"):
+    def handler(req):
+        return httpx.Response(
+            status, headers={"content-type": "text/html; charset=utf-8"}, text=html, extensions={"reason_phrase": reason}
+        )
+
+    return httpx.Client(transport=httpx.MockTransport(handler))
+
+
+def test_fetch_page_and_gateway_refusal():
+    page = fetch(page_client(), "https://example.com/")
+    assert (page.status, page.title, page.text()) == (200, "Hi", "x")
+    # The site's own 403 is content, not a gateway error.
+    assert fetch(page_client(403, b"Forbidden"), "http://example.com/").status == 403
+    with pytest.raises(GatewayError, match="No IPs match your targeting"):
+        fetch(page_client(503, b"NO_RAY"), "http://example.com/")
+    with pytest.raises(ValueError):
+        fetch(page_client(), "ftp://example.com/")
+
+
+def test_html_to_text():
+    html = """<html><head><title>T</title><style>p{}</style></head>
+    <body><script>var a=1</script><h1>Title</h1><p>One   two</p><p>Three&amp;four<br>five</p>
+    <svg><title>icon</title></svg><ul><li>a</li><li>b</li></ul></body></html>"""
+    assert html_to_text(html) == "Title\n\nOne two\n\nThree&four\nfive\n\na\n\nb"