langchain-decodo 0.1.0__tar.gz

langchain_decodo-0.1.0/.gitignore

@@ -0,0 +1,197 @@
+.vs/
+.claude/
+.idea/
+#Emacs backup
+*~
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Google GitHub Actions credentials files created by:
+# https://github.com/google-github-actions/auth
+#
+# That action recommends adding this gitignore to prevent accidentally committing keys.
+gha-creds-*.json
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.codspeed/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+notebooks/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env.*
+!.env.example
+.envrc
+*.pem
+*.key
+*.crt
+credentials.json
+# SSH private keys (no file extension, so *.key never matches them)
+id_rsa
+id_dsa
+id_ecdsa
+id_ed25519
+*_rsa
+*_dsa
+*_ecdsa
+*_ed25519
+!*.pub
+
+# Keystores
+*.p12
+*.pfx
+*.jks
+
+# Tokens, cookie jars, and git credential stores
+token.json
+Cookies
+Cookies.db
+cookies.sqlite
+cookies.txt
+.git-credentials
+.venv*
+venv*
+env/
+ENV/
+env.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.mypy_cache_test/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# macOS display setting files
+.DS_Store
+
+# Wandb directory
+wandb/
+
+# asdf tool versions
+.tool-versions
+/.ruff_cache/
+
+*.pkl
+*.bin
+
+# integration test artifacts
+data_map*
+\[('_type', 'fake'), ('stop', None)]
+
+# Replit files
+*replit*
+
+node_modules
+
+prof
+virtualenv/
+scratch/
+
+.langgraph_api/

langchain_decodo-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Decodo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain_decodo-0.1.0/Makefile

@@ -0,0 +1,39 @@
+
+.PHONY: all lint format test tests integration_tests help
+
+all: help
+
+######################
+# LINTING AND FORMATTING
+######################
+
+lint format: PYTHON_FILES=.
+lint:
+	ruff check $(PYTHON_FILES)
+	ruff format $(PYTHON_FILES) --diff
+	mypy $(PYTHON_FILES)
+
+format:
+	ruff format $(PYTHON_FILES)
+	ruff check --fix $(PYTHON_FILES)
+
+######################
+# TESTING
+######################
+
+tests test:
+	pytest tests/unit_tests
+
+integration_tests:
+	pytest tests/integration_tests
+
+######################
+# HELP
+######################
+
+help:
+	@echo '----'
+	@echo 'lint                         - run linters'
+	@echo 'format                       - run formatters'
+	@echo 'tests                        - run unit tests'
+	@echo 'integration_tests            - run integration tests (requires DECODO_API_TOKEN)'

langchain_decodo-0.1.0/PKG-INFO

@@ -0,0 +1,182 @@
+Metadata-Version: 2.5
+Name: langchain-decodo
+Version: 0.1.0
+Summary: LangChain integration for the Decodo web scraping API
+Project-URL: Homepage, https://decodo.com
+Project-URL: Documentation, https://developers.decodo.com
+Project-URL: Repository, https://github.com/langchain-ai/langchain/tree/master/libs/partners/decodo
+Author-email: Decodo <support@decodo.com>
+License: MIT
+License-File: LICENSE
+Keywords: ai,decodo,langchain,llm,rag,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: langchain-core<2.0.0,>=0.1.0
+Description-Content-Type: text/markdown
+
+# langchain-decodo
+
+LangChain partner package for the [Decodo](https://decodo.com) web scraping API.
+
+Provides two **LangChain tools** and a **document loader** that let LLM agents
+and RAG pipelines fetch live web content without managing proxies, JavaScript
+rendering, or anti-bot protection.
+
+## Installation
+
+```bash
+pip install langchain-decodo
+```
+
+## Authentication
+
+All classes read your Decodo API token from the `DECODO_API_TOKEN` environment
+variable, or you can pass it explicitly.
+
+```bash
+export DECODO_API_TOKEN="your-decodo-api-token"
+```
+
+Get a token from the [Decodo Dashboard](https://app.decodo.com).
+
+## Components
+
+### `DecodoWebScrapeTool`
+
+Scrape any URL and return its full content as markdown or plain text.
+Handles JavaScript rendering, CAPTCHAs, and geo-blocking automatically.
+
+```python
+from langchain_decodo import DecodoWebScrapeTool
+
+tool = DecodoWebScrapeTool()  # reads DECODO_API_TOKEN from env
+content = tool.run("https://example.com")
+print(content)
+```
+
+Pass explicitly:
+
+```python
+from pydantic import SecretStr
+from langchain_decodo import DecodoWebScrapeTool
+
+tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("YOUR_TOKEN"))
+```
+
+### `DecodoSearchTool`
+
+Search Google, Amazon, or Reddit and return structured JSON results.
+
+```python
+from langchain_decodo import DecodoSearchTool
+
+tool = DecodoSearchTool()
+
+# Google search (default)
+results = tool.run({"query": "LangChain latest release", "engine": "google"})
+
+# Amazon product search
+results = tool.run({"query": "Python programming book", "engine": "amazon"})
+
+# Reddit discussion search
+results = tool.run({"query": "best web scraping libraries", "engine": "reddit"})
+```
+
+Returns a JSON string — a list of objects with `content`, `url`, and
+`status_code` fields.
+
+Supported engines:
+
+| `engine` | Decodo target | Description |
+|---|---|---|
+| `google` | `google_search` | Google SERP |
+| `amazon` | `amazon_search` | Amazon product search |
+| `reddit` | `google_search` + `site:reddit.com` | Reddit via Google |
+
+### `DecodoLoader`
+
+Load one or more URLs as LangChain `Document` objects for use in RAG pipelines.
+
+```python
+from langchain_decodo import DecodoLoader
+
+loader = DecodoLoader(
+    urls=[
+        "https://python.org/about/",
+        "https://docs.python.org/3/whatsnew/3.12.html",
+    ],
+)
+docs = loader.load()
+
+for doc in docs:
+    print(doc.metadata["url"], "—", len(doc.page_content), "chars")
+```
+
+Each `Document` has:
+
+- `page_content` — scraped text/markdown.
+- `metadata["url"]` — the source URL.
+- `metadata["source"]` — same as `url` (LangChain convention).
+- `metadata["status_code"]` — HTTP status from the target site.
+
+## LangChain agent example
+
+```python
+from langchain import hub
+from langchain.agents import AgentExecutor, create_react_agent
+from langchain_openai import ChatOpenAI
+from langchain_decodo import DecodoWebScrapeTool, DecodoSearchTool
+
+tools = [DecodoWebScrapeTool(), DecodoSearchTool()]
+llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+prompt = hub.pull("hwchase17/react")
+
+agent = create_react_agent(llm=llm, tools=tools, prompt=prompt)
+executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
+
+result = executor.invoke({
+    "input": "What is the latest stable version of Python? Check python.org."
+})
+print(result["output"])
+```
+
+## RAG pipeline example
+
+```python
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain.chains import RetrievalQA
+from langchain_decodo import DecodoLoader
+
+loader = DecodoLoader(urls=["https://python.org/about/"])
+docs = loader.load()
+
+splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+chunks = splitter.split_documents(docs)
+
+store = FAISS.from_documents(chunks, OpenAIEmbeddings())
+chain = RetrievalQA.from_chain_type(
+    llm=ChatOpenAI(model="gpt-4o-mini"),
+    retriever=store.as_retriever(search_kwargs={"k": 4}),
+)
+
+result = chain.invoke({"query": "What is Python used for?"})
+print(result["result"])
+```
+
+## Links
+
+- [Decodo website](https://decodo.com)
+- [Decodo API documentation](https://developers.decodo.com)
+- [Decodo Dashboard](https://app.decodo.com)
+- [LangChain documentation](https://python.langchain.com)

langchain_decodo-0.1.0/README.md

@@ -0,0 +1,157 @@
+# langchain-decodo
+
+LangChain partner package for the [Decodo](https://decodo.com) web scraping API.
+
+Provides two **LangChain tools** and a **document loader** that let LLM agents
+and RAG pipelines fetch live web content without managing proxies, JavaScript
+rendering, or anti-bot protection.
+
+## Installation
+
+```bash
+pip install langchain-decodo
+```
+
+## Authentication
+
+All classes read your Decodo API token from the `DECODO_API_TOKEN` environment
+variable, or you can pass it explicitly.
+
+```bash
+export DECODO_API_TOKEN="your-decodo-api-token"
+```
+
+Get a token from the [Decodo Dashboard](https://app.decodo.com).
+
+## Components
+
+### `DecodoWebScrapeTool`
+
+Scrape any URL and return its full content as markdown or plain text.
+Handles JavaScript rendering, CAPTCHAs, and geo-blocking automatically.
+
+```python
+from langchain_decodo import DecodoWebScrapeTool
+
+tool = DecodoWebScrapeTool()  # reads DECODO_API_TOKEN from env
+content = tool.run("https://example.com")
+print(content)
+```
+
+Pass explicitly:
+
+```python
+from pydantic import SecretStr
+from langchain_decodo import DecodoWebScrapeTool
+
+tool = DecodoWebScrapeTool(decodo_api_token=SecretStr("YOUR_TOKEN"))
+```
+
+### `DecodoSearchTool`
+
+Search Google, Amazon, or Reddit and return structured JSON results.
+
+```python
+from langchain_decodo import DecodoSearchTool
+
+tool = DecodoSearchTool()
+
+# Google search (default)
+results = tool.run({"query": "LangChain latest release", "engine": "google"})
+
+# Amazon product search
+results = tool.run({"query": "Python programming book", "engine": "amazon"})
+
+# Reddit discussion search
+results = tool.run({"query": "best web scraping libraries", "engine": "reddit"})
+```
+
+Returns a JSON string — a list of objects with `content`, `url`, and
+`status_code` fields.
+
+Supported engines:
+
+| `engine` | Decodo target | Description |
+|---|---|---|
+| `google` | `google_search` | Google SERP |
+| `amazon` | `amazon_search` | Amazon product search |
+| `reddit` | `google_search` + `site:reddit.com` | Reddit via Google |
+
+### `DecodoLoader`
+
+Load one or more URLs as LangChain `Document` objects for use in RAG pipelines.
+
+```python
+from langchain_decodo import DecodoLoader
+
+loader = DecodoLoader(
+    urls=[
+        "https://python.org/about/",
+        "https://docs.python.org/3/whatsnew/3.12.html",
+    ],
+)
+docs = loader.load()
+
+for doc in docs:
+    print(doc.metadata["url"], "—", len(doc.page_content), "chars")
+```
+
+Each `Document` has:
+
+- `page_content` — scraped text/markdown.
+- `metadata["url"]` — the source URL.
+- `metadata["source"]` — same as `url` (LangChain convention).
+- `metadata["status_code"]` — HTTP status from the target site.
+
+## LangChain agent example
+
+```python
+from langchain import hub
+from langchain.agents import AgentExecutor, create_react_agent
+from langchain_openai import ChatOpenAI
+from langchain_decodo import DecodoWebScrapeTool, DecodoSearchTool
+
+tools = [DecodoWebScrapeTool(), DecodoSearchTool()]
+llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
+prompt = hub.pull("hwchase17/react")
+
+agent = create_react_agent(llm=llm, tools=tools, prompt=prompt)
+executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
+
+result = executor.invoke({
+    "input": "What is the latest stable version of Python? Check python.org."
+})
+print(result["output"])
+```
+
+## RAG pipeline example
+
+```python
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain.chains import RetrievalQA
+from langchain_decodo import DecodoLoader
+
+loader = DecodoLoader(urls=["https://python.org/about/"])
+docs = loader.load()
+
+splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+chunks = splitter.split_documents(docs)
+
+store = FAISS.from_documents(chunks, OpenAIEmbeddings())
+chain = RetrievalQA.from_chain_type(
+    llm=ChatOpenAI(model="gpt-4o-mini"),
+    retriever=store.as_retriever(search_kwargs={"k": 4}),
+)
+
+result = chain.invoke({"query": "What is Python used for?"})
+print(result["result"])
+```
+
+## Links
+
+- [Decodo website](https://decodo.com)
+- [Decodo API documentation](https://developers.decodo.com)
+- [Decodo Dashboard](https://app.decodo.com)
+- [LangChain documentation](https://python.langchain.com)

langchain_decodo-0.1.0/langchain_decodo/__init__.py

@@ -0,0 +1,10 @@
+from langchain_decodo._version import __version__
+from langchain_decodo.document_loaders import DecodoLoader
+from langchain_decodo.tools import DecodoSearchTool, DecodoWebScrapeTool
+
+__all__ = [
+    "DecodoLoader",
+    "DecodoSearchTool",
+    "DecodoWebScrapeTool",
+    "__version__",
+]

langchain_decodo-0.1.0/langchain_decodo/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"