langchain-oxylabs 0.1.0__tar.gz

langchain_oxylabs-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Oxylabs, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_oxylabs-0.1.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.3
+Name: langchain-oxylabs
+Version: 0.1.0
+Summary: Oxylabs integration package for LangChain
+License: MIT
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: langchain-core (>=0.3.15,<0.4.0)
+Requires-Dist: oxylabs (>=2.0.0,<3.0.0)
+Project-URL: Repository, https://github.com/oxylabs/langchain-oxylabs
+Project-URL: Release Notes, https://github.com/oxylabs/langchain-oxylabs/releases
+Project-URL: Source Code, https://github.com/oxylabs/langchain-oxylabs
+Description-Content-Type: text/markdown
+
+# langchain-oxylabs
+
+This package contains the LangChain integration with Oxylabs, providing tools to scrape Google search results 
+with Oxylabs Web Scraper API using LangChain's framework.
+
+## Installation
+
+```bash
+pip install -U langchain-oxylabs
+```
+
+## Credentials
+Create your API user credentials: Sign up for a free trial or purchase the product
+in the [Oxylabs dashboard](https://dashboard.oxylabs.io/en/registration)
+to create your API user credentials.
+
+Configure your Oxylabs credentials by setting the following environment variables:
+- `OXYLABS_USERNAME` - Your Oxylabs API username
+- `OXYLABS_PASSWORD` - Your Oxylabs API password
+
+## Usage
+`langchain_oxylabs` package provides the following classes:
+- `OxylabsSearchRun` - A tool that returns scraped Google search results in a formatted text
+- `OxylabsSearchResults` - A tool that returns scraped Google search results in a JSON format
+- `OxylabsSearchAPIWrapper` - An API wrapper for initializing Oxylabs API
+
+Here is an example usage of these classes:
+
+```python
+import json
+from langchain_oxylabs import OxylabsSearchRun, OxylabsSearchResults, OxylabsSearchAPIWrapper
+
+# Initialize the API wrapper
+oxylabs_wrapper = OxylabsSearchAPIWrapper()
+
+# Initialize the search run tool
+run_tool = OxylabsSearchRun(wrapper=oxylabs_wrapper)
+
+# Invoke the tool and print results
+results_text = run_tool.invoke({"query": "Visit restaurants in Vilnius."})
+print(results_text)
+
+# Initialize the search results tool
+results_tool = OxylabsSearchResults(wrapper=oxylabs_wrapper)
+
+# Invoke the tool and print results
+response_results = results_tool.invoke({"query": "Visit restaurants in Paris."})
+response_results = json.loads(response_results)
+for result in response_results:
+    for key, value in result.items():
+        print(f"{key}: {value}")
+```
+
+## License
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+

langchain_oxylabs-0.1.0/README.md

@@ -0,0 +1,56 @@
+# langchain-oxylabs
+
+This package contains the LangChain integration with Oxylabs, providing tools to scrape Google search results 
+with Oxylabs Web Scraper API using LangChain's framework.
+
+## Installation
+
+```bash
+pip install -U langchain-oxylabs
+```
+
+## Credentials
+Create your API user credentials: Sign up for a free trial or purchase the product
+in the [Oxylabs dashboard](https://dashboard.oxylabs.io/en/registration)
+to create your API user credentials.
+
+Configure your Oxylabs credentials by setting the following environment variables:
+- `OXYLABS_USERNAME` - Your Oxylabs API username
+- `OXYLABS_PASSWORD` - Your Oxylabs API password
+
+## Usage
+`langchain_oxylabs` package provides the following classes:
+- `OxylabsSearchRun` - A tool that returns scraped Google search results in a formatted text
+- `OxylabsSearchResults` - A tool that returns scraped Google search results in a JSON format
+- `OxylabsSearchAPIWrapper` - An API wrapper for initializing Oxylabs API
+
+Here is an example usage of these classes:
+
+```python
+import json
+from langchain_oxylabs import OxylabsSearchRun, OxylabsSearchResults, OxylabsSearchAPIWrapper
+
+# Initialize the API wrapper
+oxylabs_wrapper = OxylabsSearchAPIWrapper()
+
+# Initialize the search run tool
+run_tool = OxylabsSearchRun(wrapper=oxylabs_wrapper)
+
+# Invoke the tool and print results
+results_text = run_tool.invoke({"query": "Visit restaurants in Vilnius."})
+print(results_text)
+
+# Initialize the search results tool
+results_tool = OxylabsSearchResults(wrapper=oxylabs_wrapper)
+
+# Invoke the tool and print results
+response_results = results_tool.invoke({"query": "Visit restaurants in Paris."})
+response_results = json.loads(response_results)
+for result in response_results:
+    for key, value in result.items():
+        print(f"{key}: {value}")
+```
+
+## License
+This project is licensed under the MIT License - see the LICENSE file for details.
+

langchain_oxylabs-0.1.0/langchain_oxylabs/__init__.py

@@ -0,0 +1,10 @@
+from langchain_oxylabs.tools import OxylabsSearchRun, OxylabsSearchResults
+from langchain_oxylabs.utilities import OxylabsSearchAPIWrapper
+from langchain_oxylabs._version import __version__
+
+__all__ = [
+    "OxylabsSearchRun",
+    "OxylabsSearchResults",
+    "OxylabsSearchAPIWrapper",
+    "__version__",
+]

langchain_oxylabs-0.1.0/langchain_oxylabs/_version.py

@@ -0,0 +1,8 @@
+from importlib import metadata
+
+try:
+    __version__ = metadata.version(__package__)
+except metadata.PackageNotFoundError:
+    # Case where package metadata is not available.
+    __version__ = ""
+del metadata  # optional, avoids polluting the results of dir(__package__)

langchain_oxylabs-0.1.0/langchain_oxylabs/tools.py

@@ -0,0 +1,323 @@
+"""Tool for the Oxylabs Search API."""
+
+import json
+from typing import Optional, Type
+from langchain_core.tools import BaseTool
+from pydantic import BaseModel, ConfigDict, Field
+from langchain_oxylabs.utilities import OxylabsSearchAPIWrapper
+from langchain_core.callbacks import (
+    AsyncCallbackManagerForToolRun,
+    CallbackManagerForToolRun,
+)
+
+
+class OxylabsSearchQueryInput(BaseModel):
+    """Input for the OxylabsSearch tool."""
+
+    query: str = Field(description="query to retrieve on Oxylabs Search API")
+    geo_location: Optional[str] = Field(
+        default="California,United States",
+        description="Geographic location for the search;"
+        " adjust if location-specific information is requested.",
+    )
+
+
+class OxylabsSearchRun(BaseTool):  # type: ignore[override, override]
+    """Oxylabs Search Run tool.
+
+    Setup:
+        Install ``langchain-oxylabs``,
+        and set environment variables ``OXYLABS_USERNAME`` and ``OXYLABS_PASSWORD``.
+
+        .. code-block:: bash
+
+            pip install langchain-oxylabs
+            export OXYLABS_USERNAME="your-oxylabs-username"
+            export OXYLABS_PASSWORD="your-oxylabs-password"
+
+    Instantiation:
+        .. code-block:: python
+
+            from langchain_oxylabs import OxylabsSearchRun, OxylabsSearchAPIWrapper
+
+            oxylabs_wrapper = OxylabsSearchAPIWrapper()
+            tool = OxylabsSearchRun(wrapper=oxylabs_wrapper)
+
+    Invocation with args:
+        .. code-block:: python
+
+            tool_.invoke({"query": "Visit restaurants in Vilnius."})
+
+    Invocation with ToolCall:
+
+        .. code-block:: python
+
+            tool = OxylabsSearchRun(wrapper=oxylabs_wrapper, kwargs={"result_categories": ["local_information", "combined_search_result"]})
+            model_generated_tool_call = {"args": {"query": "Visit restaurants in Vilnius.", "geo_location": "Vilnius,Lithuania"}, "id": "1", "name": "oxylabs_search", "type": "tool_call",}
+            tool.invoke(model_generated_tool_call)
+
+        .. code-block:: python
+
+            ToolMessage(
+                ('  LOCAL PACK: \n'
+                 '  ITEMS ITEMS: \n'
+                 '    ITEMS-ITEM-1: \n'
+                 '    CID: 13950149882539119249\n'
+                 '    POS: 1\n'
+                 '    TITLE: Etno Dvaras\n'
+                 '    RATING: 4.5\n'
+                 '    ADDRESS: Lithuanian\n'
+                 '    ITEMS-ITEM-2: \n'
+                 '    CID: 711702509070991018\n'
+                 '    POS: 2\n'
+                 '    TITLE: Lokys\n'
+                 '    RATING: 4.5\n'
+                 '    ADDRESS: Lithuanian\n'
+                 '    ITEMS-ITEM-3: \n'
+                 '    CID: 7630589473191639738\n'
+                 '    POS: 3\n'
+                 '    TITLE: Senoji trobelė\n'
+                 '    RATING: 4.4\n'
+                 '    ADDRESS: Lithuanian\n'
+                 '\n'
+                 '  ORGANIC RESULTS ITEMS: \n'
+                 '    ORGANIC-ITEM-1: \n'
+                 '    POS: 1\n'
+                 '    URL: '
+                 'https://www.tripadvisor.com/Restaurants-g274951-Vilnius_Vilnius_County.html\n'
+                 '    DESC: Some of the best restaurants in Vilnius for families with children '
+                 'include: Momo grill Vilnius · Jurgis ir Drakonas Ogmios · RoseHip Vegan\xa0'
+                 '...\n'
+                 '    TITLE: THE 10 BEST Restaurants in Vilnius (Updated November ...\n'
+                 '    SITELINKS: \n'
+                 '      SITELINKS: \n'
+                 '      INLINE ITEMS: \n'
+                 '        INLINE-ITEM-1: \n'
+                 '        URL: '
+                 'https://www.tripadvisor.com/Restaurants-g274951-zfp58-Vilnius_Vilnius_County.html\n'
+                 '        TITLE: Vilnius Dinner Restaurants\n'
+                 '        INLINE-ITEM-2: \n'
+                 '        URL: '
+                 'https://www.tripadvisor.com/Restaurants-g274951-zfp2-Vilnius_Vilnius_County.html\n'
+                 '        TITLE: Vilnius Breakfast Restaurants\n'
+                 '        INLINE-ITEM-3: \n'
+                 '        URL: '
+                 'https://www.tripadvisor.com/Restaurants-g274951-c8-Vilnius_Vilnius_County.html\n'
+                 '        TITLE: Cafés in Vilnius\n'
+                 '    URL_SHOWN: https://www.tripadvisor.com› ... › Vilnius\n'
+                 '    FAVICON_TEXT: Tripadvisor\n'
+                 '    ORGANIC-ITEM-2: \n'
+                 '    POS: 2\n'
+                 '    URL: '
+                 'https://theweek.com/culture-life/food-drink/foodie-guide-to-vilnius-lithuania\n'
+                 "    DESC: Jun 24, 2024 — Lithuania's capital has established itself as an "
+                 "affordable culinary hotspot as four of the city's restaurants awarded "
+                 'Michelin stars.\n'
+                 '    TITLE: Star quality: a foodie guide to Vilnius\n'
+                 '    URL_SHOWN: https://theweek.com› Culture & Life › Food & Drink\n'
+                 '    FAVICON_TEXT: The Week\n'
+                 '    ...
+            )
+
+    """  # noqa: E501
+
+    name: str = "oxylabs_search"
+    description: str = (
+        "A meta search engine."
+        "Ideal for situations where you need to answer questions about current events,"
+        "facts, products, recipes, local information, and other topics "
+        "that can be explored via web browsing. "
+        "The input should be a search query and, if applicable,"
+        " a geo_location string to enhance result accuracy. "
+        "The output is a compiled, formatted summary of query results. "
+    )
+    wrapper: OxylabsSearchAPIWrapper
+    kwargs: dict = Field(default_factory=dict)
+    args_schema: Type[BaseModel] = OxylabsSearchQueryInput
+
+    def _run(
+        self,
+        query: str,
+        geo_location: Optional[str] = "",
+        # run_manager is added to enable tracing
+        run_manager: Optional[CallbackManagerForToolRun] = None
+    ) -> str:
+        """Use the tool."""
+
+        kwargs_ = {**self.kwargs}
+        kwargs_.update(
+            {
+                "geo_location": geo_location,
+            }
+        )
+
+        return self.wrapper.run(query, **kwargs_)
+
+    async def _arun(
+        self,
+        query: str,
+        geo_location: Optional[str] = "",
+        # run_manager is added to enable tracing
+        run_manager: Optional[AsyncCallbackManagerForToolRun] = None
+    ) -> str:
+        """Use the tool asynchronously."""
+
+        kwargs_ = {**self.kwargs}
+        kwargs_.update(
+            {
+                "geo_location": geo_location,
+            }
+        )
+
+        return await self.wrapper.arun(query, **kwargs_)
+
+
+class OxylabsSearchResults(BaseTool):  # type: ignore[override, override]
+    """Oxylabs Search Results tool.
+
+    Setup:
+        Install ``langchain-oxylabs``,
+        and set environment variables ``OXYLABS_USERNAME`` and ``OXYLABS_PASSWORD``.
+
+        .. code-block:: bash
+
+            pip install langchain-oxylabs
+            export OXYLABS_USERNAME="your-oxylabs-username"
+            export OXYLABS_PASSWORD="your-oxylabs-password"
+
+    Instantiation:
+        .. code-block:: python
+
+            from langchain_oxylabs import OxylabsSearchRun, OxylabsSearchAPIWrapper
+
+            oxylabs_wrapper = OxylabsSearchAPIWrapper()
+            tool = OxylabsSearchResults(wrapper=oxylabs_wrapper)
+
+    Invocation with args:
+        .. code-block:: python
+
+            response_results = tool.invoke({"query": "Visit restaurants in Vilnius."})
+            response_results = json.loads(response_results)
+            for item in response_results:
+                print(item)
+
+    Invocation with ToolCall:
+
+        .. code-block:: python
+
+            model_generated_tool_call = {"args": {"query": "Visit restaurants in Vilnius.", "geo_location": "Vilnius,Lithuania"}, "id": "1", "name": "oxylabs_search", "type": "tool_call",}
+            tool.invoke(model_generated_tool_call)
+
+        .. code-block:: python
+
+            ToolMessage('[{"paid": [], "organic": [{"pos": 1, "url": '
+             '"https://www.tripadvisor.com/Restaurants-g274951-Vilnius_Vilnius_County.html", '
+             '"desc": "Some of the best restaurants in Vilnius for families with children '
+             'include: Momo grill Vilnius \\u00b7 Jurgis ir Drakonas Ogmios \\u00b7 '
+             'RoseHip Vegan\\u00a0...", "title": "THE 10 BEST Restaurants in Vilnius '
+             '(Updated November ...", "sitelinks": {"inline": [{"url": '
+             '"https://www.tripadvisor.com/Restaurants-g274951-zfp58-Vilnius_Vilnius_County.html", '
+             '"title": "Vilnius Dinner Restaurants"}, {"url": '
+             '"https://www.tripadvisor.com/Restaurants-g274951-zfp2-Vilnius_Vilnius_County.html", '
+             '"title": "Vilnius Breakfast Restaurants"}, {"url": '
+             '"https://www.tripadvisor.com/Restaurants-g274951-c8-Vilnius_Vilnius_County.html", '
+             '"title": "Caf\\u00e9s in Vilnius"}]}, "url_shown": '
+             '"https://www.tripadvisor.com\\u203a ... \\u203a Vilnius", "pos_overall": 1, '
+             '"favicon_text": "Tripadvisor"}, {"pos": 2, "url": '
+             '"https://www.amsterdamfoodie.nl/2022/foodie-guide-to-vilnius-lithuania/", '
+             '"desc": "Dec 2, 2022 \\u2014 Vilnius takes the top spot for most creative '
+             'dining with the fabulous restaurant Amandus. Modern gastronomy, with a nod '
+             'to Lithuanian tradition, goes hand in\\u00a0...", "title": "A Foodie\'s '
+             'Guide to Vilnius, Lithuania", "url_shown": '
+             '"https://www.amsterdamfoodie.nl\\u203a blog", "pos_overall": 2, '
+             '"favicon_text": "Amsterdam Foodie"}, {"pos": 3, "url": '
+             '"https://www.reddit.com/r/Vilnius/comments/uudd1p/good_restaurants_street_food_in_vilnius/", '
+             '"desc": "- Alin\\u0117 Lei\\u010diai - also traditional peasant food of '
+             'history but in a more upscale environment, and more expensive, table service '
+             'and more diverse\\u00a0...", "title": "Good restaurants & street food in '
+             'Vilnius?", "url_shown": "10+ comments  \\u00b7  2 years ago", "pos_overall": '
+             '3, "favicon_text": "Reddit\\u00a0\\u00b7\\u00a0r/Vilnius"}, {"pos": 4, '
+             '"url": '
+             '"https://theweek.com/culture-life/food-drink/foodie-guide-to-vilnius-lithuania", '
+             '"desc": "Jun 24, 2024 \\u2014 Lithuania\'s capital has established itself as '
+             "an affordable culinary hotspot as four of the city's restaurants awarded "
+             'Michelin stars.", "title": "Star quality: a foodie guide to Vilnius", '
+             '"sitelinks": {"inline": [{"url": '
+             '"https://theweek.com/culture-life/food-drink/foodie-guide-to-vilnius-lithuania#:~:text=Nineteen18,-With%20its%20%60%60sleek%20and", '
+             '"title": "Nineteen18"}, {"url": '
+             '"https://theweek.com/culture-life/food-drink/foodie-guide-to-vilnius-lithuania#:~:text=D%C5%BEiaugsmas,-D%C5%BEiaugsmas", '
+             '"title": "D\\u017eiaugsmas"}, {"url": '
+             '"https://theweek.com/culture-life/food-drink/foodie-guide-to-vilnius-lithuania#:~:text=Gaspar%27s", '
+             '"title": "Gaspar\'s"}]}, "url_shown": "https://theweek.com\\u203a Culture & '
+             'Life \\u203a Food & Drink", "pos_overall": 4, "favicon_text": "The Week"}], '
+             '"local_pack": {"items": [{"cid": "13950149882539119249", "pos": 1, "title": '
+             '"Etno Dvaras", "rating": 4.5, "address": "Lithuanian", "rating_count": 0}, '
+             '{"cid": "7630589473191639738", "pos": 2, "title": "Senoji trobel\\u0117", '
+             '"rating": 4.4, "address": "Lithuanian", "rating_count": 0}, {"cid": '
+             '"6362402554387935672", "pos": 3, "title": "Ertlio Namas", "rating": 4.7, '
+             '"address": "\\u0160v. Jono g. 7", "subtitle": "Lithuanian", "rating_count": '
+             '0}], "pos_overall": 5}, "related_searches": {"pos_overall": 6, '
+             '"related_searches": ["Best restaurants in Vilnius old town", "Unique places '
+             'to eat in Vilnius", "Best visit restaurants in vilnius", "Best Lithuanian '
+             'restaurants in Vilnius", "Restaurants Vilnius old Town", "Vilnius '
+             'restaurants Michelin", "Fine dining Vilnius", "Tripadvisor Vilnius '
+             'Restaurants"]}, "search_information": {"query": "Visit restaurants in '
+             'Vilnius.", "geo_location": "United States", "showing_results_for": "Visit '
+             'restaurants in Vilnius.", "total_results_count": 1260000}, '
+             '"total_results_count": 1260000}]'
+         )
+    """  # noqa: E501
+
+    name: str = "oxylabs_search_results"
+    description: str = (
+        "A meta search engine."
+        "Ideal for situations where you need to answer questions about current events,"
+        "facts, products, recipes, local information, and other topics "
+        "that can be explored via web browsing. "
+        "The input should be a search query and, if applicable,"
+        " a geo_location string to enhance result accuracy. "
+        "The output is a JSON array of response page objects. "
+    )
+    wrapper: OxylabsSearchAPIWrapper
+    kwargs: dict = Field(default_factory=dict)
+    args_schema: Type[BaseModel] = OxylabsSearchQueryInput
+
+    model_config = ConfigDict(
+        extra="allow",
+    )
+
+    def _run(
+        self,
+        query: str,
+        geo_location: Optional[str] = "",
+        # run_manager is added to enable tracing
+        run_manager: Optional[CallbackManagerForToolRun] = None
+    ) -> str:
+        """Use the tool."""
+
+        kwargs_ = {**self.kwargs}
+        kwargs_.update(
+            {
+                "geo_location": geo_location,
+            }
+        )
+
+        return json.dumps(self.wrapper.results(query, **kwargs_))
+
+    async def _arun(
+        self,
+        query: str,
+        geo_location: Optional[str] = "",
+        # run_manager is added to enable tracing
+        run_manager: Optional[AsyncCallbackManagerForToolRun] = None
+    ) -> str:
+        """Use the tool asynchronously."""
+
+        kwargs_ = {**self.kwargs}
+        kwargs_.update(
+            {
+                "geo_location": geo_location,
+            }
+        )
+
+        return json.dumps(await self.wrapper.aresults(query, **kwargs_))

langchain_oxylabs-0.1.0/langchain_oxylabs/utilities.py

@@ -0,0 +1,688 @@
+"""Chain that calls Oxylabs API for Google Search."""
+
+import asyncio
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+from platform import python_version, architecture
+from oxylabs import RealtimeClient  # type: ignore[import-untyped]
+from langchain_core.utils import get_from_dict_or_env
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from langchain_oxylabs._version import __version__
+
+RESULT_CATEGORIES = [
+    "knowledge_graph",
+    "combined_search_result",
+    "product_information",
+    "local_information",
+    "search_information",
+]
+
+IMAGE_BINARY_CONTENT_ARRAY_ATTRIBUTE = "images"
+BINARY_CONTENT_REPLACEMENT = "Redacted base64 image string..."
+
+
+@dataclass
+class ResponseElement:
+    tag: str
+    display_tag: str
+    path_: str
+    python_type: str
+    parent: Optional["ResponseElement"]
+
+
+def _get_default_excluded_result_attributes() -> List[str]:
+    return ["pos_overall"]
+
+
+def _get_default_image_content_attributes() -> List[str]:
+    return ["image_data", "data"]
+
+
+def _get_default_params() -> dict:
+    """Provide default parameters for OxylabsSearchAPIWrapper.
+
+    Returns:
+        dict: Default parameters, including the following keys:
+            - source (str): Source of the search engine, e.g., "google_search".
+            - user_agent_type (str): User agent type, e.g.,
+                "desktop". Can be set using values from `oxylabs.utils.types`.
+            - render (str): Render type for the search results, e.g.,
+                "html". Can be set using values from `oxylabs.utils.types`.
+            - domain (str): Domain for the search engine, e.g., "com".
+            - start_page (int): Starting page number for search results.
+            - pages (int): Number of pages to retrieve.
+            - limit (int): Maximum number of results to return.
+            - parse (bool): Whether to enable result parsing into an object.
+            - locale (str): Locale or location for the search.
+            - geo_location (str): Geographic location for the search.
+            - parsing_instructions (dict): Additional instructions for parsing
+                the search results.
+            - context (list): Search context information.
+            - request_timeout (int): Timeout for the Oxylabs service, in seconds.
+            - result_categories (list): Specifies the desired categories for the
+                results from the available:
+                    `knowledge_graph` which includes Oxylabs results categories:
+                       `knowledge`, `recipes`, `carousel`, `apps`.
+                    `combined_search_result` which includes Oxylabs results categories:
+                       `organic`, `paid`, `organic_videos`, `featured_snippet`,
+                       `top_stories`, `finance`, `sports_games`, `twitter`,
+                       `discussions_and_forums`, `images`, `videos`, `video_box`.
+                    `product_information` which includes Oxylabs results categories:
+                       `popular_products`, `pla`.
+                    `local_information` which includes Oxylabs results categories:
+                       `top_sights`, `flights`, `hotels`, `local_pack`,
+                       `local_service_ads`, `jobs`.
+                    `search_information` which includes Oxylabs results categories:
+                       `search_information`, `related_searches`,
+                       `related_searches_categorized`, `related_questions`.
+                The list preserves the order of the categories,
+                allowing prioritized filtering. If left empty (default),
+                results are returned from all available categories without filtering.
+    """
+
+    return {
+        "source": "google_search",
+        "user_agent_type": "desktop",
+        "render": "html",
+        "domain": "com",
+        "start_page": 1,
+        "pages": 1,
+        "limit": 5,
+        "parse": True,
+        "locale": "",
+        "geo_location": "",
+        "parsing_instructions": {},
+        "context": [],
+        "request_timeout": 165,
+        "result_categories": [],
+    }
+
+
+class OxylabsSearchAPIWrapper(BaseModel):
+    """Wrapper class for OxylabsSearchAPI.
+
+    Example:
+        ```python
+        from langchain_oxylabs import OxylabsSearchAPIWrapper
+
+        oxylabs_api = OxylabsSearchAPIWrapper(
+            "oxylabs_username": <OXYLABS_USERNAME>,
+            "oxylabs_password": <OXYLABS_PASSWORD>,
+            params={
+                "source": "google_search",
+                "user_agent_type": "desktop",
+                "render": "html",
+                "domain": "com",
+                "start_page": 1,
+                "pages": 1,
+                "limit": 5,
+                "parse": True,
+                "locale": "",
+                "geo_location": "",
+                "parsing_instructions": {},
+                "context": [],
+                "request_timeout": 165,
+                "result_categories": [],
+            }
+        )
+        search_query = "Oxylabs"
+        result = oxylabs_api.run(search_query)
+        print(result)
+        ```
+    """
+
+    include_binary_image_data: Optional[bool] = Field(default=False)
+    parsing_recursion_depth: int = Field(default=5)
+
+    search_engine: Any = None
+    params: dict = Field(default_factory=_get_default_params)
+    result_categories: Optional[list] = Field(default=[])
+
+    excluded_result_attributes: List[str] = Field(
+        default_factory=_get_default_excluded_result_attributes
+    )
+    image_binary_content_attributes: List[str] = Field(
+        default_factory=_get_default_image_content_attributes
+    )
+    image_binary_content_array_attribute: str = Field(
+        default=IMAGE_BINARY_CONTENT_ARRAY_ATTRIBUTE
+    )
+    binary_content_replacement: str = Field(default=BINARY_CONTENT_REPLACEMENT)
+
+    oxylabs_username: Optional[str] = None
+    oxylabs_password: Optional[str] = None
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_environment(cls, values: Dict) -> Any:
+        """
+        Validate that oxylabs username,
+        password and python package exists in environment.
+        """
+
+        default_params = _get_default_params()
+        default_params.update(**values.get("params", {}))
+        current_params = default_params
+        oxylabs_username = get_from_dict_or_env(
+            values, "oxylabs_username", "OXYLABS_USERNAME"
+        )
+        oxylabs_password = get_from_dict_or_env(
+            values, "oxylabs_password", "OXYLABS_PASSWORD"
+        )
+        if not (oxylabs_username and oxylabs_password):
+            raise RuntimeError(
+                "Please set up environment variables:"
+                " `OXYLABS_USERNAME` and `OXYLABS_PASSWORD`"
+                " or provide credentials in the parameters"
+                " as `oxylabs_username` and `oxylabs_password`."
+            )
+
+        formed_values: Dict[str, Any] = {
+            "oxylabs_username": oxylabs_username,
+            "oxylabs_password": oxylabs_password,
+            "params": dict(current_params),
+            "include_binary_image_data": values.get(
+                "include_binary_image_data", False
+            ),
+            "parsing_recursion_depth": values.get(
+                "parsing_recursion_depth", 5
+            )}
+
+        if "result_categories" in formed_values["params"]:
+            validated_categories = cls.validate_response_categories(
+                formed_values["params"]["result_categories"]
+            )
+            if validated_categories:
+                formed_values["result_categories"] = validated_categories
+
+        try:
+            bits, _ = architecture()
+            oxylabs_realtime_client = RealtimeClient(
+                username=oxylabs_username,
+                password=oxylabs_password,
+                sdk_type=f"oxylabs-sdk-langchain/{__version__} ({python_version()}; {bits})",
+            )
+            # The process to set any available provider
+            source_ = formed_values["params"]["source"]
+            source_provider_map = {
+                "google": ["google_search"],
+            }
+            engine_ = ""
+            for engine, sources_ in source_provider_map.items():
+                if source_ in sources_:
+                    engine_ = engine
+
+                    break
+
+            if engine_:
+                oxylabs_realtime_client_by_provider = getattr(
+                    oxylabs_realtime_client, engine_
+                )
+                formed_values["search_engine"] = oxylabs_realtime_client_by_provider
+            else:
+                supported_sources = ", ".join(sum(source_provider_map.values(), []))
+                raise NotImplementedError(
+                    f"Source: `{source_}` is not supported."
+                    f" Supported  sources: {supported_sources}"
+                )
+
+        except NotImplementedError as exc:
+            raise NotImplementedError(f"{exc}") from exc
+
+        except Exception as exc:
+            raise RuntimeError(f"Unknown Oxylabs Python SDK integration error: {exc}") from exc
+
+        return formed_values
+
+    @staticmethod
+    def validate_response_categories(result_categories: list) -> list:
+        validated_categories = []
+        for result_category in result_categories:
+            if result_category in RESULT_CATEGORIES:
+                validated_categories.append(result_category)
+
+        return validated_categories
+
+    async def arun(self, query: str, **kwargs: Any) -> str:
+        """
+        Run query through OxylabsSearchAPI and parse result async.
+        """
+        return self._process_response(await self.aresults(query, **kwargs), **kwargs)
+
+    def run(self, query: str, **kwargs: Any) -> str:
+        """
+        Run query through OxylabsSearchAPI and parse result.
+        """
+        return self._process_response(self.results(query, **kwargs), **kwargs)
+
+    def results(self, query: str, **kwargs: Any) -> List[Dict[str, Any]]:
+        """
+        Run query through Oxylabs Web Scrapper API and return SERPResponse object.
+        """
+        params_ = self.get_params(**kwargs)
+        search_client = self.search_engine
+        search_result = search_client.scrape_search(query, **params_)
+
+        try:
+            validated_responses = self._validate_response(search_result)
+
+        except RuntimeError:
+            return []
+
+        return validated_responses
+
+    async def aresults(self, query: str, **kwargs: Any) -> List[Dict[str, Any]]:
+        """
+        Run query through Oxylabs Web Scrapper API and return SERPResponse object async.
+        """
+        params_ = self.get_params(**kwargs)
+
+        search_client = self.search_engine
+        search_result = await asyncio.to_thread(
+            search_client.scrape_search,
+            query,
+            **params_,
+        )
+
+        try:
+            validated_responses = self._validate_response(search_result)
+
+        except RuntimeError:
+            return []
+
+        return validated_responses
+
+    def get_params(self, **kwargs: Any) -> Dict[str, Any]:
+        """
+        Get default configuration parameters for OxylabsSearchAPI for scrape_search().
+        """
+        wrapper_params_ = ["result_categories"]
+
+        _params = {
+            f"{p_key}": self.params[p_key]
+            for p_key in self.params
+            if self.params[p_key] and p_key not in wrapper_params_
+        }
+
+        for key, value in kwargs.items():
+            if key in _params:
+                _params[key] = value
+
+        return _params
+
+    @staticmethod
+    def _validate_response(response: Any) -> List[Dict[Any, Any]]:
+        """
+        Validate Oxylabs SERPResponse format and unpack data.
+        """
+        validated_results = []
+        try:
+            result_pages = response.raw["results"]
+            if not isinstance(result_pages, list) or not result_pages:
+                raise ValueError("No results returned!")
+
+            for result_page in result_pages:
+                result_page = dict(result_page)
+                content = result_page["content"]
+                if not isinstance(content, dict):
+                    raise ValueError(
+                        "Result `content` format error,"
+                        " try setting parameter `parse` to True"
+                    )
+
+                unpacked_results = content["results"]
+
+                if not isinstance(unpacked_results, dict):
+                    raise ValueError("Response format Error!")
+
+                if unpacked_results:
+                    validated_results.append(unpacked_results)
+
+            return validated_results
+
+        except (KeyError, IndexError, TypeError, ValueError) as exc:
+            raise RuntimeError(f"Response Validation Error: {str(exc)}") from exc
+
+    def _process_response(self, res: Any, **kwargs: Any) -> str:
+        """
+        Process Oxylabs SERPResponse and serialize search results to string.
+        """
+
+        result_ = "No good search result found"
+
+        result_category_processing_map = {
+            "knowledge_graph": self._create_knowledge_graph_snippets,
+            "combined_search_result": self._create_combined_search_result_snippets,
+            "product_information": self._create_product_information_snippets,
+            "local_information": self._create_local_information_snippets,
+            "search_information": self._create_search_information_snippets,
+        }
+
+        snippets: List[str] = []
+        validated_categories = self.validate_response_categories(
+            kwargs.get("result_categories", [])
+        )
+        result_categories_ = validated_categories or self.result_categories or []
+
+        for _, validated_response in enumerate(res):
+            if result_categories_:
+                for result_category in result_categories_:
+                    result_category_processing_map[result_category](
+                        validated_response, snippets
+                    )
+            else:
+                for result_category in result_category_processing_map:
+                    result_category_processing_map[result_category](
+                        validated_response, snippets
+                    )
+
+        # Combine all snippets
+        if snippets:
+            result_ = "\n\n".join(snippets)
+
+        return result_
+
+    def recursive_snippet_collector(
+        self,
+        target_structure: Any,
+        max_depth: int,
+        current_depth: int,
+        parent_: ResponseElement,
+    ) -> str:
+        target_snippets: List[str] = []
+
+        padding_multiplier = current_depth + 1
+        recursion_padding = "  " * padding_multiplier
+
+        if current_depth >= max_depth:
+            return "\n".join(target_snippets)
+
+        if isinstance(target_structure, (str, float, int)):
+            self.recursion_process_simple_types(
+                parent_, recursion_padding, target_snippets, target_structure
+            )
+
+        elif isinstance(target_structure, dict):
+            self.recursion_process_dict(
+                current_depth,
+                max_depth,
+                parent_,
+                recursion_padding,
+                target_snippets,
+                target_structure,
+            )
+
+        elif isinstance(target_structure, (list, tuple)):
+            self.recursion_process_array(
+                current_depth,
+                max_depth,
+                parent_,
+                recursion_padding,
+                target_snippets,
+                target_structure,
+            )
+
+        return "\n".join(target_snippets)
+
+    def recursion_process_array(
+        self,
+        current_depth: int,
+        max_depth: int,
+        parent_: ResponseElement,
+        recursion_padding: str,
+        target_snippets: list,
+        target_structure: Any,
+    ) -> None:
+        if target_structure:
+            target_snippets.append(
+                f"{recursion_padding}{parent_.display_tag.upper()} ITEMS: "
+            )
+        for nr_, element_ in enumerate(target_structure):
+            target_snippets.append(
+                self.recursive_snippet_collector(
+                    element_,
+                    max_depth=max_depth,
+                    current_depth=current_depth + 1,
+                    parent_=ResponseElement(
+                        path_=f"{parent_.path_.upper()}-ITEM-{nr_ + 1}",
+                        tag=parent_.tag.upper(),
+                        display_tag=f"{parent_.tag.upper()}-ITEM-{nr_ + 1}",
+                        python_type=str(type(target_structure)),
+                        parent=parent_,
+                    ),
+                )
+            )
+
+    def recursion_process_dict(
+        self,
+        current_depth: int,
+        max_depth: int,
+        parent_: ResponseElement,
+        recursion_padding: str,
+        target_snippets: list,
+        target_structure: Any,
+    ) -> None:
+        if target_structure:
+            target_snippets.append(
+                f"{recursion_padding}{parent_.display_tag.upper()}: "
+            )
+            for key_, value_ in target_structure.items():
+                if isinstance(value_, dict):
+                    if value_:
+                        target_snippets.append(f"{recursion_padding}{key_.upper()}: ")
+                        target_snippets.append(
+                            self.recursive_snippet_collector(
+                                value_,
+                                max_depth=max_depth,
+                                current_depth=current_depth + 1,
+                                parent_=ResponseElement(
+                                    path_=f"{parent_.path_.upper()}-{key_.upper()}",
+                                    tag=key_.upper(),
+                                    display_tag=key_.upper(),
+                                    python_type=str(type(value_)),
+                                    parent=parent_,
+                                ),
+                            )
+                        )
+
+                elif isinstance(value_, (list, tuple)):
+                    if value_:
+                        target_snippets.append(
+                            f"{recursion_padding}{key_.upper()} ITEMS: "
+                        )
+                        for nr_, _element in enumerate(value_):
+                            target_snippets.append(
+                                self.recursive_snippet_collector(
+                                    _element,
+                                    max_depth=max_depth,
+                                    current_depth=current_depth + 1,
+                                    parent_=ResponseElement(
+                                        path_=f"{parent_.path_.upper()}"
+                                        f"-{key_.upper()}-ITEM-{nr_ + 1}",
+                                        tag=key_.upper(),
+                                        display_tag=f"{key_.upper()}-ITEM-{nr_ + 1}",
+                                        python_type=str(type(value_)),
+                                        parent=parent_,
+                                    ),
+                                )
+                            )
+
+                elif isinstance(value_, (str, float, int)):
+                    if value_:
+                        if (
+                            key_ in self.image_binary_content_attributes
+                            and not self.include_binary_image_data
+                        ):
+                            value_ = self.binary_content_replacement
+
+                        if key_ not in self.excluded_result_attributes:
+                            target_snippets.append(
+                                f"{recursion_padding}{key_.upper()}: {str(value_)}"
+                            )
+
+    def recursion_process_simple_types(
+        self,
+        parent_: ResponseElement,
+        recursion_padding: str,
+        target_snippets: list,
+        target_structure: Any,
+    ) -> None:
+        if target_structure:
+            if parent_.python_type == str(type([])):
+                if (
+                    self.image_binary_content_array_attribute.upper()
+                    in parent_.path_.split("-")[-3:]
+                    or parent_.tag.lower() in self.image_binary_content_attributes
+                ) and not self.include_binary_image_data:
+                    target_structure = self.binary_content_replacement
+
+                target_snippets.append(
+                    f"{recursion_padding}{parent_.display_tag}: {str(target_structure)}"
+                )
+
+            elif parent_.python_type == str(type({})):
+                if (
+                    parent_.tag.lower() in self.image_binary_content_attributes
+                    and not self.include_binary_image_data
+                ):
+                    target_structure = self.binary_content_replacement
+
+                if parent_.tag.lower() not in self.excluded_result_attributes:
+                    target_snippets.append(
+                        f"{recursion_padding}{parent_.display_tag}:"
+                        f" {str(target_structure)}"
+                    )
+
+    def process_tags(
+        self, snippets_: list, tags_: list, results: dict, group_name: str = ""
+    ) -> None:
+        check_tags = [tag_[0] in results for tag_ in tags_]
+        if any(check_tags):
+            for tag in tags_:
+                tag_content = results.get(tag[0], {}) or {}
+                if tag_content:
+                    collected_snippets = self.recursive_snippet_collector(
+                        tag_content,
+                        max_depth=self.parsing_recursion_depth,
+                        current_depth=0,
+                        parent_=ResponseElement(
+                            path_=f"{group_name}-{tag[0]}",
+                            tag=tag[0],
+                            display_tag=tag[1],
+                            python_type=str((type(tag_content))),
+                            parent=None,
+                        ),
+                    )
+                    if collected_snippets:
+                        snippets_.append(collected_snippets)
+
+    def _create_knowledge_graph_snippets(
+        self, results: dict, knowledge_graph_snippets: list
+    ) -> None:
+        """
+        Create knowledge graph snippets
+        from Oxylabs SERPResponse data search_information tag.
+        """
+
+        knowledge_graph_tags = [
+            ("knowledge", "Knowledge Graph"),
+            ("recipes", "Recipes"),
+            ("item_carousel", "Item Carousel"),
+            ("apps", "Apps"),
+        ]
+        self.process_tags(
+            knowledge_graph_snippets, knowledge_graph_tags, results, "Knowledge"
+        )
+
+    def _create_combined_search_result_snippets(
+        self, results: dict, combined_search_result_snippets: list
+    ) -> None:
+        """
+        Create combined search result snippets
+        from Oxylabs SERPResponse data search_information tag.
+        """
+
+        combined_search_result_tags = [
+            ("organic", "Organic Results"),
+            ("organic_videos", "Organic Videos"),
+            ("paid", "Paid Results"),
+            ("featured_snipped", "Feature Snipped"),
+            ("top_stories", "Top Stories"),
+            ("finance", "Finance"),
+            ("sports_games", "Sports Games"),
+            ("twitter", "Twitter"),
+            ("discussions_and_forums", "Discussions and Forums"),
+            ("images", "Images"),
+            ("videos", "Videos"),
+            ("video_box", "Video box"),
+        ]
+        self.process_tags(
+            combined_search_result_snippets,
+            combined_search_result_tags,
+            results,
+            "Combined Search Results",
+        )
+
+    def _create_product_information_snippets(
+        self, results: dict, product_information_snippets: list
+    ) -> None:
+        """
+        Create product information snippets from
+        Oxylabs SERPResponse data search_information tag.
+        """
+
+        product_information_tags = [
+            ("popular_products", "Popular Products"),
+            ("pla", "Product Listing Ads (PLA)"),
+        ]
+        self.process_tags(
+            product_information_snippets,
+            product_information_tags,
+            results,
+            "Product Information",
+        )
+
+    def _create_local_information_snippets(
+        self, results: dict, local_information_snippets: list
+    ) -> None:
+        """
+        Create local group information snippets
+        from Oxylabs SERPResponse data search_information tag.
+        """
+
+        local_information_tags = [
+            ("top_sights", "Top Sights"),
+            ("flights", "Flights"),
+            ("hotels", "Hotels"),
+            ("local_pack", "Local Pack"),
+            ("local_service_ads", "Local Service Ads"),
+            ("jobs", "Jobs"),
+        ]
+        self.process_tags(
+            local_information_snippets,
+            local_information_tags,
+            results,
+            "Local Information",
+        )
+
+    def _create_search_information_snippets(
+        self, results: dict, search_information_snippets: list
+    ) -> None:
+        """
+        Create search information snippets
+        from Oxylabs SERPResponse data search_information tag.
+        """
+
+        search_information_tags = [
+            ("search_information", "Search Information"),
+            ("related_searches", "Related Searches"),
+            ("related_searches_categorized", "Related Searches Categorized"),
+            ("related_questions", "Related Questions"),
+        ]
+        self.process_tags(
+            search_information_snippets,
+            search_information_tags,
+            results,
+            "Search Information",
+        )

langchain_oxylabs-0.1.0/pyproject.toml

@@ -0,0 +1,69 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "langchain-oxylabs"
+version = "0.1.0"
+description = "Oxylabs integration package for LangChain"
+authors = []
+readme = "README.md"
+repository = "https://github.com/oxylabs/langchain-oxylabs"
+license = "MIT"
+
+[tool.mypy]
+disallow_untyped_defs = "True"
+
+[tool.poetry.urls]
+"Source Code" = "https://github.com/oxylabs/langchain-oxylabs"
+"Release Notes" = "https://github.com/oxylabs/langchain-oxylabs/releases"
+
+[tool.poetry.dependencies]
+python = ">=3.9,<4.0"
+langchain-core = "^0.3.15"
+oxylabs = "^2.0.0"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "T201"]
+
+[tool.coverage.run]
+omit = ["tests/*"]
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers --strict-config --durations=5"
+asyncio_mode = "auto"
+
+[tool.poetry.group.test]
+optional = true
+
+[tool.poetry.group.codespell]
+optional = true
+
+[tool.poetry.group.test_integration]
+optional = true
+
+[tool.poetry.group.lint]
+optional = true
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+
+[tool.poetry.group.test.dependencies]
+pytest = "^7.4.3"
+pytest-asyncio = "^0.23.2"
+pytest-socket = "^0.7.0"
+pytest-watcher = "^0.3.4"
+langchain-tests = "^0.3.5"
+
+[tool.poetry.group.codespell.dependencies]
+codespell = "^2.2.6"
+
+[tool.poetry.group.test_integration.dependencies]
+
+[tool.poetry.group.lint.dependencies]
+ruff = "^0.5"
+
+[tool.poetry.group.typing.dependencies]
+mypy = "^1.10"