serperdev-haystack 1.0.0__tar.gz

serperdev_haystack-1.0.0/.gitignore

@@ -0,0 +1,146 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+volumes/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# IDEs
+.vscode
+
+# Docs generation artifacts
+_readme_*.md
+.idea
+
+# macOS
+.DS_Store
+
+# http cache (requests-cache)
+**/http_cache.sqlite
+
+# ruff
+.ruff_cache

serperdev_haystack-1.0.0/LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023-present deepset GmbH
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

serperdev_haystack-1.0.0/PKG-INFO

@@ -0,0 +1,40 @@
+Metadata-Version: 2.4
+Name: serperdev-haystack
+Version: 1.0.0
+Summary: Haystack integration for SerperDev Web Search
+Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/serperdev#readme
+Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
+Project-URL: Source, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/serperdev
+Author-email: deepset GmbH <info@deepset.ai>
+License-Expression: Apache-2.0
+License-File: LICENSE.txt
+Keywords: AI Search,Haystack,Serper,SerperDev,Web Search
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.10
+Requires-Dist: haystack-ai>=2.24.1
+Requires-Dist: httpx>=0.27.0
+Description-Content-Type: text/markdown
+
+# serperdev-haystack
+
+[![PyPI - Version](https://img.shields.io/pypi/v/serperdev-haystack.svg)](https://pypi.org/project/serperdev-haystack)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/serperdev-haystack.svg)](https://pypi.org/project/serperdev-haystack)
+
+- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/serperdev/CHANGELOG.md)
+
+---
+
+## Contributing
+
+Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
+
+To run integration tests locally, you need to export the `SERPERDEV_API_KEY` environment variable.

serperdev_haystack-1.0.0/README.md

@@ -0,0 +1,14 @@
+# serperdev-haystack
+
+[![PyPI - Version](https://img.shields.io/pypi/v/serperdev-haystack.svg)](https://pypi.org/project/serperdev-haystack)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/serperdev-haystack.svg)](https://pypi.org/project/serperdev-haystack)
+
+- [Changelog](https://github.com/deepset-ai/haystack-core-integrations/blob/main/integrations/serperdev/CHANGELOG.md)
+
+---
+
+## Contributing
+
+Refer to the general [Contribution Guidelines](https://github.com/deepset-ai/haystack-core-integrations/blob/main/CONTRIBUTING.md).
+
+To run integration tests locally, you need to export the `SERPERDEV_API_KEY` environment variable.

serperdev_haystack-1.0.0/pydoc/config_docusaurus.yml

@@ -0,0 +1,13 @@
+loaders:
+  - modules:
+      - haystack_integrations.components.websearch.serperdev.websearch
+    search_path: [../src]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  description: SerperDev integration for Haystack
+  id: integrations-serperdev
+  filename: serperdev.md
+  title: SerperDev

serperdev_haystack-1.0.0/pyproject.toml

@@ -0,0 +1,165 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "serperdev-haystack"
+dynamic = ["version"]
+description = "Haystack integration for SerperDev Web Search"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+keywords = ["Haystack", "SerperDev", "Serper", "Web Search", "AI Search"]
+authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
+classifiers = [
+  "License :: OSI Approved :: Apache Software License",
+  "Development Status :: 4 - Beta",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+]
+dependencies = ["haystack-ai>=2.24.1", "httpx>=0.27.0"]
+
+[project.urls]
+Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/serperdev#readme"
+Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues"
+Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/serperdev"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/haystack_integrations"]
+
+[tool.hatch.version]
+source = "vcs"
+tag-pattern = 'integrations\/serperdev-v(?P<version>.*)'
+
+[tool.hatch.version.raw-options]
+root = "../.."
+git_describe_command = 'git describe --tags --match="integrations/serperdev-v[0-9]*"'
+
+[tool.hatch.envs.default]
+installer = "uv"
+dependencies = ["haystack-pydoc-tools", "ruff"]
+
+[tool.hatch.envs.default.scripts]
+docs = ["haystack-pydoc pydoc/config_docusaurus.yml"]
+fmt = "ruff check --fix {args}; ruff format {args}"
+fmt-check = "ruff check {args} && ruff format --check {args}"
+
+[tool.hatch.envs.test]
+dependencies = [
+    "pytest",
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-rerunfailures",
+    "mypy",
+    "pip",
+]
+
+[tool.hatch.envs.test.scripts]
+unit = 'pytest -m "not integration" {args:tests}'
+integration = 'pytest -m "integration" {args:tests}'
+all = 'pytest {args:tests}'
+unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
+integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
+types = "mypy -p haystack_integrations.components.websearch.serperdev {args}"
+
+[tool.mypy]
+install_types = true
+non_interactive = true
+check_untyped_defs = true
+disallow_incomplete_defs = true
+
+[[tool.mypy.overrides]]
+module = ["httpx.*"]
+ignore_missing_imports = true
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "A",
+    "ANN",
+    "ARG",
+    "B",
+    "C",
+    "D102",   # Missing docstring in public method
+    "D103",   # Missing docstring in public function
+    "D205",   # 1 blank line required between summary line and description
+    "D209",   # Closing triple quotes go to new line
+    "D213",   # summary lines must be positioned on the second physical line of the docstring
+    "D417",   # Missing argument descriptions in the docstring
+    "D419",   # Docstring is empty
+    "DTZ",
+    "E",
+    "EM",
+    "F",
+    "I",
+    "ICN",
+    "ISC",
+    "N",
+    "PLC",
+    "PLE",
+    "PLR",
+    "PLW",
+    "Q",
+    "RUF",
+    "S",
+    "T",
+    "TID",
+    "UP",
+    "W",
+    "YTT",
+]
+ignore = [
+    # Allow non-abstract empty methods in abstract base classes
+    "B027",
+    # Allow function calls in argument defaults (common Haystack pattern for Secret.from_env_var)
+    "B008",
+    # Ignore checks for possible passwords
+    "S105",
+    "S106",
+    "S107",
+    # Ignore complexity
+    "C901",
+    "PLR0911",
+    "PLR0912",
+    "PLR0913",
+    "PLR0915",
+    # Allow `Any` type - used legitimately for dynamic types and SDK boundaries
+    "ANN401",
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["haystack_integrations"]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "parents"
+
+[tool.ruff.lint.per-file-ignores]
+# Tests can use magic values, assertions, relative imports, and don't need type annotations
+"tests/**/*" = ["PLR2004", "S101", "TID252", "D", "ANN"]
+
+[tool.coverage.run]
+source = ["haystack_integrations"]
+branch = true
+relative_files = true
+parallel = false
+
+[tool.coverage.report]
+omit = ["*/tests/*", "*/__init__.py"]
+show_missing = true
+exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"]
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers"
+markers = [
+  "integration: integration tests",
+]
+log_cli = true
+asyncio_default_fixture_loop_scope = "function"

serperdev_haystack-1.0.0/src/haystack_integrations/components/websearch/serperdev/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack_integrations.components.websearch.serperdev.websearch import (
+    SerperDevError,
+    SerperDevWebSearch,
+)
+
+__all__ = ["SerperDevError", "SerperDevWebSearch"]

serperdev_haystack-1.0.0/src/haystack_integrations/components/websearch/serperdev/websearch.py

@@ -0,0 +1,280 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+from haystack import ComponentError, Document, component, default_from_dict, default_to_dict, logging
+from haystack.utils import Secret, deserialize_secrets_inplace
+
+logger = logging.getLogger(__name__)
+
+
+SERPERDEV_BASE_URL = "https://google.serper.dev/search"
+
+
+class SerperDevError(ComponentError): ...
+
+
+@component
+class SerperDevWebSearch:
+    """
+    Uses [Serper](https://serper.dev/) to search the web for relevant documents.
+
+    See the [Serper Dev website](https://serper.dev/) for more details.
+
+    Usage example:
+    ```python
+    from haystack.utils import Secret
+
+    from haystack_integrations.components.websearch.serperdev import SerperDevWebSearch
+
+    serper_dev_api = Secret.from_env_var("SERPERDEV_API_KEY")
+
+    websearch = SerperDevWebSearch(top_k=10, api_key=serper_dev_api)
+    results = websearch.run(query="Who is the boyfriend of Olivia Wilde?")
+
+    assert results["documents"]
+    assert results["links"]
+
+    # Example with domain filtering - exclude subdomains
+    websearch_filtered = SerperDevWebSearch(
+        top_k=10,
+        allowed_domains=["example.com"],
+        exclude_subdomains=True,  # Only results from example.com, not blog.example.com
+        api_key=serper_dev_api,
+    )
+    results_filtered = websearch_filtered.run(query="search query")
+    ```
+    """
+
+    def __init__(
+        self,
+        api_key: Secret = Secret.from_env_var("SERPERDEV_API_KEY"),
+        top_k: int | None = 10,
+        allowed_domains: list[str] | None = None,
+        search_params: dict[str, Any] | None = None,
+        *,
+        exclude_subdomains: bool = False,
+    ) -> None:
+        """
+        Initialize the SerperDevWebSearch component.
+
+        :param api_key: API key for the Serper API.
+        :param top_k: Number of documents to return.
+        :param allowed_domains: List of domains to limit the search to.
+        :param exclude_subdomains: Whether to exclude subdomains when filtering by allowed_domains.
+            If True, only results from the exact domains in allowed_domains will be returned.
+            If False, results from subdomains will also be included. Defaults to False.
+        :param search_params: Additional parameters passed to the Serper API.
+            For example, you can set 'num' to 20 to increase the number of search results.
+            See the [Serper website](https://serper.dev/) for more details.
+        """
+        self.api_key = api_key
+        self.top_k = top_k
+        self.allowed_domains = allowed_domains
+        self.exclude_subdomains = exclude_subdomains
+        self.search_params = search_params or {}
+
+        # Ensure that the API key is resolved.
+        _ = self.api_key.resolve_value()
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+                Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            top_k=self.top_k,
+            allowed_domains=self.allowed_domains,
+            exclude_subdomains=self.exclude_subdomains,
+            search_params=self.search_params,
+            api_key=self.api_key.to_dict(),
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SerperDevWebSearch":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        deserialize_secrets_inplace(data["init_parameters"], keys=["api_key"])
+        return default_from_dict(cls, data)
+
+    def _is_domain_allowed(self, url: str) -> bool:
+        """
+        Check if a URL's domain is allowed based on allowed_domains and exclude_subdomains settings.
+
+        :param url: The URL to check.
+        :returns: True if the domain is allowed, False otherwise.
+        """
+        if not self.allowed_domains:
+            return True
+
+        try:
+            parsed = urlparse(url)
+            domain = parsed.netloc.lower()
+
+            for allowed_domain in self.allowed_domains:
+                allowed_domain_lower = allowed_domain.lower()
+
+                if self.exclude_subdomains:
+                    # Exact domain match only
+                    if domain == allowed_domain_lower:
+                        return True
+                # Allow subdomains (current behavior)
+                elif domain == allowed_domain_lower or domain.endswith("." + allowed_domain_lower):
+                    return True
+
+            return False
+        except Exception:
+            # If URL parsing fails, allow the result to be safe
+            return True
+
+    @component.output_types(documents=list[Document], links=list[str])
+    def run(self, query: str) -> dict[str, list[Document] | list[str]]:
+        """
+        Use [Serper](https://serper.dev/) to search the web.
+
+        :param query: Search query.
+        :returns: A dictionary with the following keys:
+            - "documents": List of documents returned by the search engine.
+            - "links": List of links returned by the search engine.
+        :raises SerperDevError: If an error occurs while querying the SerperDev API.
+        :raises TimeoutError: If the request to the SerperDev API times out.
+        """
+        payload, headers = self._prepare_request(query)
+        try:
+            response = httpx.post(SERPERDEV_BASE_URL, headers=headers, json=payload, timeout=30)
+            response.raise_for_status()  # Will raise an HTTPError for bad responses
+        except httpx.TimeoutException as error:
+            msg = f"Request to {self.__class__.__name__} timed out."
+            raise TimeoutError(msg) from error
+
+        except httpx.HTTPStatusError as e:
+            msg = f"An error occurred while querying {self.__class__.__name__}. Error: {e}, Response: {e.response.text}"
+            raise SerperDevError(msg) from e
+
+        except httpx.HTTPError as e:
+            msg = f"An error occurred while querying {self.__class__.__name__}. Error: {e}"
+            raise SerperDevError(msg) from e
+
+        documents, links = self._parse_response(response)
+
+        logger.debug(
+            "Serper Dev returned {number_documents} documents for the query '{query}'",
+            number_documents=len(documents),
+            query=query,
+        )
+        return {"documents": documents[: self.top_k], "links": links[: self.top_k]}
+
+    @component.output_types(documents=list[Document], links=list[str])
+    async def run_async(self, query: str) -> dict[str, list[Document] | list[str]]:
+        """
+        Asynchronously uses [Serper](https://serper.dev/) to search the web.
+
+        This is the asynchronous version of the `run` method with the same parameters and return values.
+
+        :param query: Search query.
+        :returns: A dictionary with the following keys:
+            - "documents": List of documents returned by the search engine.
+            - "links": List of links returned by the search engine.
+        :raises SerperDevError: If an error occurs while querying the SerperDev API.
+        :raises TimeoutError: If the request to the SerperDev API times out.
+        """
+        payload, headers = self._prepare_request(query)
+        try:
+            async with httpx.AsyncClient() as client:
+                response = await client.post(SERPERDEV_BASE_URL, headers=headers, json=payload, timeout=30)
+            response.raise_for_status()  # Will raise an HTTPError for bad responses
+        except httpx.TimeoutException as error:
+            msg = f"Request to {self.__class__.__name__} timed out."
+            raise TimeoutError(msg) from error
+
+        except httpx.HTTPStatusError as e:
+            msg = f"An error occurred while querying {self.__class__.__name__}. Error: {e}, Response: {e.response.text}"
+            raise SerperDevError(msg) from e
+
+        except httpx.HTTPError as e:
+            msg = f"An error occurred while querying {self.__class__.__name__}. Error: {e}"
+            raise SerperDevError(msg) from e
+
+        documents, links = self._parse_response(response)
+
+        logger.debug(
+            "Serper Dev returned {number_documents} documents for the query '{query}'",
+            number_documents=len(documents),
+            query=query,
+        )
+        return {"documents": documents[: self.top_k], "links": links[: self.top_k]}
+
+    def _prepare_request(self, query: str) -> tuple[dict[str, Any], dict[str, str]]:
+        query_prepend = "OR ".join(f"site:{domain} " for domain in self.allowed_domains) if self.allowed_domains else ""
+        payload = {"q": query_prepend + query, "gl": "us", "hl": "en", "autocorrect": True, **self.search_params}
+        if (api_key := self.api_key.resolve_value()) is None:
+            msg = "API key cannot be `None`."
+            raise ValueError(msg)
+        headers = {"X-API-KEY": api_key}
+        return payload, headers
+
+    def _parse_response(self, response: httpx.Response) -> tuple[list[Document], list[str]]:
+        # If we reached this point, it means the request was successful and we can proceed
+        json_result = response.json()
+
+        # we get the snippet from the json result and put it in the content field of the document
+        organic = [
+            Document(meta={k: v for k, v in d.items() if k != "snippet"}, content=d.get("snippet"))
+            for d in json_result["organic"]
+            if self._is_domain_allowed(d.get("link", ""))
+        ]
+
+        # answer box is what search engine shows as a direct answer to the query
+        answer_box = []
+        if "answerBox" in json_result:
+            answer_dict = json_result["answerBox"]
+            highlighted_answers = answer_dict.get("snippetHighlighted")
+            answer_box_content = None
+            # Check if highlighted_answers is a list and has at least one element
+            if isinstance(highlighted_answers, list) and len(highlighted_answers) > 0:
+                answer_box_content = highlighted_answers[0]
+            elif isinstance(highlighted_answers, str):
+                answer_box_content = highlighted_answers
+            if not answer_box_content:
+                for key in ["snippet", "answer", "title"]:
+                    if key in answer_dict:
+                        answer_box_content = answer_dict[key]
+                        break
+            if answer_box_content and self._is_domain_allowed(answer_dict.get("link", "")):
+                answer_box = [
+                    Document(
+                        content=answer_box_content,
+                        meta={"title": answer_dict.get("title", ""), "link": answer_dict.get("link", "")},
+                    )
+                ]
+
+        # these are related questions that search engine shows
+        people_also_ask = []
+        if "peopleAlsoAsk" in json_result:
+            for result in json_result["peopleAlsoAsk"]:
+                if self._is_domain_allowed(result.get("link", "")):
+                    title = result.get("title", "")
+                    people_also_ask.append(
+                        Document(
+                            content=result["snippet"] if result.get("snippet") else title,
+                            meta={"title": title, "link": result.get("link", None)},
+                        )
+                    )
+
+        documents = answer_box + organic + people_also_ask
+
+        links = [result["link"] for result in json_result["organic"] if self._is_domain_allowed(result.get("link", ""))]
+        return documents, links

serperdev_haystack-1.0.0/tests/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

serperdev_haystack-1.0.0/tests/test_websearch.py

@@ -0,0 +1,441 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import copy
+import os
+from collections.abc import Generator
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import pytest
+from haystack import Document
+from haystack.utils.auth import Secret
+from httpx import ConnectTimeout, HTTPStatusError, ReadTimeout, Request, RequestError, Response
+
+from haystack_integrations.components.websearch.serperdev import SerperDevError, SerperDevWebSearch
+
+EXAMPLE_SERPERDEV_RESPONSE = {
+    "searchParameters": {
+        "q": "Who is the boyfriend of Olivia Wilde?",
+        "gl": "us",
+        "hl": "en",
+        "autocorrect": True,
+        "type": "search",
+    },
+    "organic": [
+        {
+            "title": "Olivia Wilde embraces Jason Sudeikis amid custody battle, Harry Styles split - Page Six",
+            "link": "https://pagesix.com/2023/01/29/olivia-wilde-hugs-it-out-with-jason-sudeikis-after-harry-styles-split/",
+            "snippet": "Looks like Olivia Wilde and Jason Sudeikis are starting 2023 on good terms. Amid their highly "
+            "publicized custody battle - and the actress' ...",
+            "date": "Jan 29, 2023",
+            "position": 1,
+        },
+        {
+            "title": "Olivia Wilde Is 'Quietly Dating' Again Following Harry Styles Split: 'He Makes Her Happy'",
+            "link": "https://www.yahoo.com/now/olivia-wilde-quietly-dating-again-183844364.html",
+            "snippet": "Olivia Wilde is “quietly dating again” following her November 2022 split from Harry Styles, "
+            "a source exclusively tells Life & Style.",
+            "date": "Feb 10, 2023",
+            "position": 2,
+        },
+        {
+            "title": "Olivia Wilde and Harry Styles' Relationship Timeline: The Way They Were - Us Weekly",
+            "link": "https://www.usmagazine.com/celebrity-news/pictures/olivia-wilde-and-harry-styles-relationship-timeline/",
+            "snippet": "Olivia Wilde started dating Harry Styles after ending her years-long engagement to Jason "
+            "Sudeikis — see their relationship timeline.",
+            "date": "Mar 10, 2023",
+            "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSgTcalNFvptTbYBiDXX55s8yCGfn6F1qbed9DAN16LvynTr9GayK5SPmY&s",
+            "position": 3,
+        },
+        {
+            "title": "Olivia Wilde Is 'Ready to Date Again' After Harry Styles Split - Us Weekly",
+            "link": "https://www.usmagazine.com/celebrity-news/news/olivia-wilde-is-ready-to-date-again-after-harry-styles-split/",
+            "snippet": "Ready for love! Olivia Wilde is officially back on the dating scene following her split from "
+            "her ex-boyfriend, Harry Styles.",
+            "date": "Mar 1, 2023",
+            "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRCRAeRy5sVE631ZctzbzuOF70xkIOHaTvh2K7dYvdiVBwALiKrIjpscok&s",
+            "position": 4,
+        },
+        {
+            "title": "Harry Styles and Olivia Wilde's Definitive Relationship Timeline - Harper's Bazaar",
+            "link": "https://www.harpersbazaar.com/celebrity/latest/a35172115/harry-styles-olivia-wilde-relationship-timeline/",
+            "snippet": "November 2020: News breaks about Olivia splitting from fiancé Jason Sudeikis. ... "
+            "In mid-November, news breaks of Olivia Wilde's split from Jason ...",
+            "date": "Feb 23, 2023",
+            "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRRqw3fvZOIGHEepxCc7yFAWYsS_v_1H6X-4nxyFJxdfRuFQw_BrI6JVzI&s",
+            "position": 5,
+        },
+        {
+            "title": "Harry Styles and Olivia Wilde's Relationship Timeline - People",
+            "link": "https://people.com/music/harry-styles-olivia-wilde-relationship-timeline/",
+            "snippet": "Harry Styles and Olivia Wilde first met on the set of Don't Worry Darling and stepped out as "
+            "a couple in January 2021. Relive all their biggest relationship ...",
+            "position": 6,
+        },
+        {
+            "title": "Jason Sudeikis and Olivia Wilde's Relationship Timeline - People",
+            "link": "https://people.com/movies/jason-sudeikis-olivia-wilde-relationship-timeline/",
+            "snippet": "Jason Sudeikis and Olivia Wilde ended their engagement of seven years in 2020. Here's a "
+            "complete timeline of their relationship.",
+            "date": "Mar 24, 2023",
+            "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSleZoXusQyJJe2WMgIuck_cVaJ8AE0_hU2QxsXzYvKANi55UQlv82yAVI&s",
+            "position": 7,
+        },
+        {
+            "title": "Olivia Wilde's anger at ex-boyfriend Harry Styles: She resents him and thinks he was using her "
+            "| Marca",
+            "link": "https://www.marca.com/en/lifestyle/celebrities/2023/02/23/63f779a4e2704e8d988b4624.html",
+            "snippet": "The two started dating after Wilde split up with actor Jason Sudeikisin 2020. However, their "
+            "relationship came to an end last November.",
+            "date": "Feb 23, 2023",
+            "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQBgJF2mSnIWCvPrqUqM4WTI9xPNWPyLvHuune85swpB1yE_G8cy_7KRh0&s",
+            "position": 8,
+        },
+        {
+            "title": "Olivia Wilde's dating history: Who has the actress dated? | The US Sun",
+            "link": "https://www.the-sun.com/entertainment/5221040/olivia-wildes-dating-history/",
+            "snippet": "AMERICAN actress Olivia Wilde started dating Harry Styles in January 2021 after breaking off "
+            "her engagement the year prior.",
+            "date": "Nov 19, 2022",
+            "imageUrl": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTpm8BToVFHJoH6yRggg0fLocLT9mt6lwsnRxFFDNdDGhDydzQiSKZ9__g&s",
+            "position": 9,
+        },
+    ],
+    "relatedSearches": [
+        {"query": "Harry Styles girlfriends in order"},
+        {"query": "Harry Styles and Olivia Wilde engaged"},
+        {"query": "Harry Styles and Olivia Wilde wedding"},
+        {"query": "Who is Harry Styles married to"},
+        {"query": "Jason Sudeikis Olivia Wilde relationship"},
+        {"query": "Olivia Wilde and Jason Sudeikis kids"},
+        {"query": "Olivia Wilde children"},
+        {"query": "Harry Styles and Olivia Wilde age difference"},
+        {"query": "Jason Sudeikis Olivia Wilde, Harry Styles"},
+    ],
+}
+
+HTTPX_PATH = "haystack_integrations.components.websearch.serperdev.websearch.httpx"
+
+
+@pytest.fixture
+def mock_serper_dev_search_result() -> Generator[MagicMock, None, None]:
+    with patch(HTTPX_PATH) as mock_run:
+        mock_run.post.return_value = Mock(status_code=200, json=lambda: EXAMPLE_SERPERDEV_RESPONSE)
+        yield mock_run
+
+
+@pytest.fixture
+def mock_serper_dev_search_result_async() -> Generator[MagicMock, None, None]:
+    with patch(f"{HTTPX_PATH}.AsyncClient") as mock_run:
+        mock_client = AsyncMock()
+        mock_client.post.return_value = Mock(status_code=200, json=lambda: EXAMPLE_SERPERDEV_RESPONSE)
+        mock_client.__aenter__.return_value = mock_client
+        mock_run.return_value = mock_client
+        yield mock_run
+
+
+@pytest.fixture
+def mock_serper_dev_search_result_no_snippet() -> Generator[MagicMock, None, None]:
+    resp = copy.deepcopy(EXAMPLE_SERPERDEV_RESPONSE)
+    del resp["organic"][0]["snippet"]
+    with patch(HTTPX_PATH) as mock_run:
+        mock_run.post.return_value = Mock(status_code=200, json=lambda: resp)
+        yield mock_run
+
+
+@pytest.fixture
+def mock_serper_dev_search_result_no_snippet_async() -> Generator[MagicMock, None, None]:
+    resp = copy.deepcopy(EXAMPLE_SERPERDEV_RESPONSE)
+    del resp["organic"][0]["snippet"]
+    with patch(f"{HTTPX_PATH}.AsyncClient") as mock_run:
+        mock_client = AsyncMock()
+        mock_client.post.return_value = Mock(status_code=200, json=lambda: resp)
+        mock_client.__aenter__.return_value = mock_client
+        mock_run.return_value = mock_client
+        yield mock_run
+
+
+class TestSerperDevSearchAPI:
+    def test_init_fail_wo_api_key(self, monkeypatch):
+        monkeypatch.delenv("SERPERDEV_API_KEY", raising=False)
+        with pytest.raises(ValueError, match=r"None of the .* environment variables are set"):
+            SerperDevWebSearch()
+
+    def test_to_dict(self, monkeypatch):
+        monkeypatch.setenv("SERPERDEV_API_KEY", "test-api-key")
+        component = SerperDevWebSearch(top_k=10, allowed_domains=["test.com"], search_params={"param": "test"})
+        data = component.to_dict()
+        assert data == {
+            "type": ("haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch"),
+            "init_parameters": {
+                "api_key": {"env_vars": ["SERPERDEV_API_KEY"], "strict": True, "type": "env_var"},
+                "top_k": 10,
+                "allowed_domains": ["test.com"],
+                "exclude_subdomains": False,
+                "search_params": {"param": "test"},
+            },
+        }
+
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("SERPERDEV_API_KEY", "test-api-key")
+        data = {
+            "type": ("haystack_integrations.components.websearch.serperdev.websearch.SerperDevWebSearch"),
+            "init_parameters": {
+                "api_key": {"env_vars": ["SERPERDEV_API_KEY"], "strict": True, "type": "env_var"},
+                "top_k": 10,
+                "allowed_domains": ["test.com"],
+                "exclude_subdomains": True,
+                "search_params": {"param": "test"},
+            },
+        }
+        component = SerperDevWebSearch.from_dict(data)
+        assert component.api_key == Secret.from_env_var("SERPERDEV_API_KEY")
+        assert component.top_k == 10
+        assert component.allowed_domains == ["test.com"]
+        assert component.exclude_subdomains is True
+        assert component.search_params == {"param": "test"}
+
+    @pytest.mark.parametrize("top_k", [1, 5, 7])
+    @pytest.mark.usefixtures("mock_serper_dev_search_result")
+    def test_web_search_top_k(self, top_k: int) -> None:
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), top_k=top_k)
+        results = ws.run(query="Who is the boyfriend of Olivia Wilde?")
+        documents = results["documents"]
+        links = results["links"]
+        assert len(documents) == len(links) == top_k
+        assert all(isinstance(doc, Document) for doc in documents)
+        assert all(isinstance(link, str) for link in links)
+        assert all(link.startswith("http") for link in links)
+
+    @pytest.mark.parametrize("top_k", [1, 5, 7])
+    @pytest.mark.asyncio
+    @pytest.mark.usefixtures("mock_serper_dev_search_result_async")
+    async def test_web_search_top_k_async(self, top_k: int) -> None:
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), top_k=top_k)
+        results = await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
+        documents = results["documents"]
+        links = results["links"]
+        assert len(documents) == len(links) == top_k
+        assert all(isinstance(doc, Document) for doc in documents)
+        assert all(isinstance(link, str) for link in links)
+        assert all(link.startswith("http") for link in links)
+
+    @pytest.mark.usefixtures("mock_serper_dev_search_result_no_snippet")
+    def test_no_snippet(self) -> None:
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), top_k=1)
+        ws.run(query="Who is the boyfriend of Olivia Wilde?")
+
+    @pytest.mark.asyncio
+    @pytest.mark.usefixtures("mock_serper_dev_search_result_no_snippet_async")
+    async def test_no_snippet_async(self) -> None:
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), top_k=1)
+        await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
+
+    @pytest.mark.parametrize("timeout_exception", [ConnectTimeout, ReadTimeout])
+    @patch("httpx.post")
+    def test_timeout_error(self, mock_post: MagicMock, timeout_exception: type[Exception]) -> None:
+        mock_post.side_effect = timeout_exception("Request has timed out.")
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"))
+
+        with pytest.raises(TimeoutError):
+            ws.run(query="Who is the boyfriend of Olivia Wilde?")
+
+    @pytest.mark.parametrize("timeout_exception", [ConnectTimeout, ReadTimeout])
+    @pytest.mark.asyncio
+    @patch("httpx.AsyncClient.post")
+    async def test_timeout_error_async(self, mock_post: AsyncMock, timeout_exception: type[Exception]) -> None:
+        mock_post.side_effect = timeout_exception("Request has timed out.")
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"))
+
+        with pytest.raises(TimeoutError):
+            await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
+
+    @patch("httpx.post")
+    def test_request_exception(self, mock_post: MagicMock) -> None:
+        mock_post.side_effect = RequestError("An errors has occurred in the request.")
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"))
+
+        with pytest.raises(SerperDevError):
+            ws.run(query="Who is the boyfriend of Olivia Wilde?")
+
+    @pytest.mark.asyncio
+    @patch("httpx.AsyncClient.post")
+    async def test_request_exception_async(self, mock_post: MagicMock) -> None:
+        mock_post.side_effect = RequestError("An errors has occurred in the request.")
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"))
+
+        with pytest.raises(SerperDevError):
+            await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
+
+    @patch("httpx.post")
+    def test_bad_response_code(self, mock_post: MagicMock) -> None:
+        mock_response = mock_post.return_value
+        mock_response.status_code = 404
+        mock_error_request = Request("POST", "https://example.com")
+        mock_error_response = Response(404)
+        mock_response.raise_for_status.side_effect = HTTPStatusError(
+            "404 Not Found.", request=mock_error_request, response=mock_error_response
+        )
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"))
+
+        with pytest.raises(SerperDevError):
+            ws.run(query="Who is the boyfriend of Olivia Wilde?")
+
+    @pytest.mark.asyncio
+    @patch("httpx.AsyncClient")
+    async def test_bad_response_code_async(self, mock_run: MagicMock) -> None:
+        mock_client = AsyncMock()
+        mock_response = Mock(status_code=404)
+        mock_error_request = Request("POST", "https://example.com")
+        mock_error_response = Response(404)
+        mock_response.raise_for_status.side_effect = HTTPStatusError(
+            "404 Not Found.", request=mock_error_request, response=mock_error_response
+        )
+        mock_client.post.return_value = mock_response
+        mock_client.__aenter__.return_value = mock_client
+        mock_run.return_value = mock_client
+        ws = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"))
+
+        with pytest.raises(SerperDevError):
+            await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
+
+    @pytest.mark.skipif(
+        not os.environ.get("SERPERDEV_API_KEY", None),
+        reason="Export an env var called SERPERDEV_API_KEY containing the SerperDev API key to run this test.",
+    )
+    @pytest.mark.integration
+    def test_web_search(self) -> None:
+        ws = SerperDevWebSearch(top_k=10)
+        results = ws.run(query="Who is the boyfriend of Olivia Wilde?")
+        documents = results["documents"]
+        links = results["links"]
+        # documents can also include answer box and "people also ask" results, so only links
+        # (which come from organic results) are guaranteed to be at most top_k
+        assert 0 < len(documents) <= 10
+        assert 0 < len(links) <= 10
+        assert all(isinstance(doc, Document) for doc in documents)
+        assert all(isinstance(link, str) for link in links)
+        assert all(link.startswith("http") for link in links)
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.environ.get("SERPERDEV_API_KEY", None),
+        reason="Export an env var called SERPERDEV_API_KEY containing the SerperDev API key to run this test.",
+    )
+    @pytest.mark.integration
+    async def test_web_search_async(self) -> None:
+        ws = SerperDevWebSearch(top_k=10)
+        results = await ws.run_async(query="Who is the boyfriend of Olivia Wilde?")
+        documents = results["documents"]
+        links = results["links"]
+        # documents can also include answer box and "people also ask" results, so only links
+        # (which come from organic results) are guaranteed to be at most top_k
+        assert 0 < len(documents) <= 10
+        assert 0 < len(links) <= 10
+        assert all(isinstance(doc, Document) for doc in documents)
+        assert all(isinstance(link, str) for link in links)
+        assert all(link.startswith("http") for link in links)
+
+    def test_exclude_subdomains_filtering(self, monkeypatch):
+        """Test that exclude_subdomains parameter properly filters results."""
+        monkeypatch.setenv("SERPERDEV_API_KEY", "test-api-key")
+
+        # Mock response with mixed domains and subdomains
+        mock_response = {
+            "organic": [
+                {
+                    "title": "Main domain result",
+                    "link": "https://example.com/page1",
+                    "snippet": "Content from main domain",
+                },
+                {
+                    "title": "Subdomain result 1",
+                    "link": "https://blog.example.com/post1",
+                    "snippet": "Content from blog subdomain",
+                },
+                {
+                    "title": "Subdomain result 2",
+                    "link": "https://shop.example.com/product1",
+                    "snippet": "Content from shop subdomain",
+                },
+                {
+                    "title": "Different domain result",
+                    "link": "https://other.com/page1",
+                    "snippet": "Content from different domain",
+                },
+            ]
+        }
+
+        with patch(HTTPX_PATH) as mock_requests:
+            mock_requests.post.return_value = Mock(status_code=200, json=lambda: mock_response)
+
+            # Test with exclude_subdomains=False (default behavior)
+            ws_include = SerperDevWebSearch(
+                api_key=Secret.from_token("test-api-key"), allowed_domains=["example.com"], exclude_subdomains=False
+            )
+            results_include = ws_include.run(query="test query")
+
+            # Should include main domain and subdomains but exclude other domains
+            assert len(results_include["documents"]) == 3  # example.com + 2 subdomains
+            assert len(results_include["links"]) == 3
+
+            included_links = results_include["links"]
+            assert "https://example.com/page1" in included_links
+            assert "https://blog.example.com/post1" in included_links
+            assert "https://shop.example.com/product1" in included_links
+            assert "https://other.com/page1" not in included_links
+
+            # Test with exclude_subdomains=True
+            ws_exclude = SerperDevWebSearch(
+                api_key=Secret.from_token("test-api-key"), allowed_domains=["example.com"], exclude_subdomains=True
+            )
+            results_exclude = ws_exclude.run(query="test query")
+
+            # Should only include main domain, exclude subdomains and other domains
+            assert len(results_exclude["documents"]) == 1  # only example.com
+            assert len(results_exclude["links"]) == 1
+
+            excluded_links = results_exclude["links"]
+            assert "https://example.com/page1" in excluded_links
+            assert "https://blog.example.com/post1" not in excluded_links
+            assert "https://shop.example.com/product1" not in excluded_links
+            assert "https://other.com/page1" not in excluded_links
+
+    def test_is_domain_allowed_helper_method(self, monkeypatch):
+        """Test the _is_domain_allowed helper method directly."""
+        monkeypatch.setenv("SERPERDEV_API_KEY", "test-api-key")
+
+        # Test with exclude_subdomains=False
+        ws_include = SerperDevWebSearch(
+            api_key=Secret.from_token("test-api-key"),
+            allowed_domains=["example.com", "test.org"],
+            exclude_subdomains=False,
+        )
+
+        # Should allow main domains and subdomains
+        assert ws_include._is_domain_allowed("https://example.com/page") is True
+        assert ws_include._is_domain_allowed("https://blog.example.com/post") is True
+        assert ws_include._is_domain_allowed("https://shop.example.com/product") is True
+        assert ws_include._is_domain_allowed("https://test.org/page") is True
+        assert ws_include._is_domain_allowed("https://sub.test.org/page") is True
+        assert ws_include._is_domain_allowed("https://other.com/page") is False
+
+        # Test with exclude_subdomains=True
+        ws_exclude = SerperDevWebSearch(
+            api_key=Secret.from_token("test-api-key"),
+            allowed_domains=["example.com", "test.org"],
+            exclude_subdomains=True,
+        )
+
+        # Should only allow exact domain matches
+        assert ws_exclude._is_domain_allowed("https://example.com/page") is True
+        assert ws_exclude._is_domain_allowed("https://blog.example.com/post") is False
+        assert ws_exclude._is_domain_allowed("https://shop.example.com/product") is False
+        assert ws_exclude._is_domain_allowed("https://test.org/page") is True
+        assert ws_exclude._is_domain_allowed("https://sub.test.org/page") is False
+        assert ws_exclude._is_domain_allowed("https://other.com/page") is False
+
+        # Test with no allowed_domains (should allow all)
+        ws_no_filter = SerperDevWebSearch(api_key=Secret.from_token("test-api-key"), allowed_domains=None)
+        assert ws_no_filter._is_domain_allowed("https://any.domain.com/page") is True