decodo-crewai 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- decodo_crewai-1.0.0/.env.example +2 -0
- decodo_crewai-1.0.0/.github/workflows/ci.yml +37 -0
- decodo_crewai-1.0.0/.github/workflows/pr-title.yml +16 -0
- decodo_crewai-1.0.0/.github/workflows/release.yml +63 -0
- decodo_crewai-1.0.0/.gitignore +40 -0
- decodo_crewai-1.0.0/CONTRIBUTING.md +25 -0
- decodo_crewai-1.0.0/LICENSE +21 -0
- decodo_crewai-1.0.0/PKG-INFO +182 -0
- decodo_crewai-1.0.0/README.md +134 -0
- decodo_crewai-1.0.0/examples/research_crew/README.md +95 -0
- decodo_crewai-1.0.0/examples/research_crew/main.py +99 -0
- decodo_crewai-1.0.0/pyproject.toml +66 -0
- decodo_crewai-1.0.0/src/decodo_crewai/__init__.py +7 -0
- decodo_crewai-1.0.0/src/decodo_crewai/_base.py +109 -0
- decodo_crewai-1.0.0/src/decodo_crewai/ecommerce_tool.py +95 -0
- decodo_crewai-1.0.0/src/decodo_crewai/py.typed +0 -0
- decodo_crewai-1.0.0/src/decodo_crewai/search_tool.py +60 -0
- decodo_crewai-1.0.0/src/decodo_crewai/serp_tool.py +67 -0
- decodo_crewai-1.0.0/src/decodo_crewai/web_scraper_tool.py +48 -0
- decodo_crewai-1.0.0/tests/test_tools.py +302 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
|
|
6
|
+
permissions:
|
|
7
|
+
contents: read
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.12", "3.13"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v7
|
|
17
|
+
|
|
18
|
+
- uses: actions/setup-python@v7
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python-version }}
|
|
21
|
+
|
|
22
|
+
- name: Install
|
|
23
|
+
run: pip install build -e ".[dev]"
|
|
24
|
+
|
|
25
|
+
- name: Ruff
|
|
26
|
+
run: |
|
|
27
|
+
ruff check .
|
|
28
|
+
ruff format --check .
|
|
29
|
+
|
|
30
|
+
- name: Mypy
|
|
31
|
+
run: mypy
|
|
32
|
+
|
|
33
|
+
- name: Pytest
|
|
34
|
+
run: pytest
|
|
35
|
+
|
|
36
|
+
- name: Build
|
|
37
|
+
run: python -m build
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
name: PR title
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
types: [opened, edited, synchronize, reopened]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
pull-requests: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
conventional:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: amannn/action-semantic-pull-request@v6
|
|
15
|
+
env:
|
|
16
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- main
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: release
|
|
13
|
+
cancel-in-progress: false
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
release:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
environment: pypi
|
|
19
|
+
permissions:
|
|
20
|
+
contents: write
|
|
21
|
+
id-token: write
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v7
|
|
24
|
+
with:
|
|
25
|
+
ref: ${{ github.ref_name }}
|
|
26
|
+
fetch-depth: 0
|
|
27
|
+
|
|
28
|
+
- run: git reset --hard ${{ github.sha }}
|
|
29
|
+
|
|
30
|
+
- uses: actions/setup-python@v7
|
|
31
|
+
with:
|
|
32
|
+
python-version: "3.12"
|
|
33
|
+
|
|
34
|
+
- name: Install
|
|
35
|
+
run: pip install build -e ".[dev]"
|
|
36
|
+
|
|
37
|
+
- name: Ruff
|
|
38
|
+
run: |
|
|
39
|
+
ruff check .
|
|
40
|
+
ruff format --check .
|
|
41
|
+
|
|
42
|
+
- name: Mypy
|
|
43
|
+
run: mypy
|
|
44
|
+
|
|
45
|
+
- name: Pytest
|
|
46
|
+
run: pytest
|
|
47
|
+
|
|
48
|
+
- id: release
|
|
49
|
+
uses: python-semantic-release/python-semantic-release@v10.7.0
|
|
50
|
+
with:
|
|
51
|
+
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
52
|
+
commit: "false"
|
|
53
|
+
changelog: "false"
|
|
54
|
+
|
|
55
|
+
- name: Build
|
|
56
|
+
if: steps.release.outputs.released == 'true'
|
|
57
|
+
run: python -m build
|
|
58
|
+
|
|
59
|
+
- name: Publish
|
|
60
|
+
if: steps.release.outputs.released == 'true'
|
|
61
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
62
|
+
with:
|
|
63
|
+
packages-dir: dist/
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.eggs/
|
|
11
|
+
*.egg
|
|
12
|
+
|
|
13
|
+
# Virtual environments
|
|
14
|
+
.venv/
|
|
15
|
+
venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Type checking / linting caches
|
|
19
|
+
.mypy_cache/
|
|
20
|
+
.ruff_cache/
|
|
21
|
+
|
|
22
|
+
# Testing
|
|
23
|
+
.pytest_cache/
|
|
24
|
+
.coverage
|
|
25
|
+
htmlcov/
|
|
26
|
+
|
|
27
|
+
# IDE
|
|
28
|
+
.idea/
|
|
29
|
+
.vscode/
|
|
30
|
+
*.swp
|
|
31
|
+
*.swo
|
|
32
|
+
|
|
33
|
+
# Environment
|
|
34
|
+
.env
|
|
35
|
+
|
|
36
|
+
# OS
|
|
37
|
+
.DS_Store
|
|
38
|
+
|
|
39
|
+
# Example output
|
|
40
|
+
examples/**/report.md
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
## Development
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
python3.12 -m venv .venv
|
|
7
|
+
.venv/bin/pip install -e ".[dev]"
|
|
8
|
+
.venv/bin/ruff check . && .venv/bin/ruff format --check .
|
|
9
|
+
.venv/bin/mypy
|
|
10
|
+
.venv/bin/pytest
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The live test calls the real API and only runs when `DECODO_LIVE_API_KEY` is set:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
DECODO_LIVE_API_KEY="<api_key>" .venv/bin/pytest -m live
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Releases
|
|
20
|
+
|
|
21
|
+
Releases are automated. Merging to `main` publishes a new version based on the
|
|
22
|
+
PR title, which must follow [Conventional Commits](https://www.conventionalcommits.org):
|
|
23
|
+
`fix:` → patch, `feat:` → minor, `feat!:` or a `BREAKING CHANGE:` footer → major.
|
|
24
|
+
`chore:`, `docs:`, `ci:`, `refactor:`, `test:` don't release. Don't edit the
|
|
25
|
+
version in `pyproject.toml`.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Decodo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: decodo-crewai
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Decodo Web Scraping API tools for CrewAI agents.
|
|
5
|
+
Project-URL: Homepage, https://decodo.com
|
|
6
|
+
Project-URL: Repository, https://github.com/Decodo/crewAI
|
|
7
|
+
Project-URL: Documentation, https://help.decodo.com/docs/introduction
|
|
8
|
+
Author-email: Decodo <support@decodo.com>
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 Decodo
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: agents,crewai,decodo,tools,web-scraping
|
|
32
|
+
Classifier: Intended Audience :: Developers
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
37
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
38
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
39
|
+
Requires-Python: <3.14,>=3.12
|
|
40
|
+
Requires-Dist: crewai>=1.15.9
|
|
41
|
+
Requires-Dist: decodo-sdk>=2.4.1
|
|
42
|
+
Requires-Dist: httpx>=0.27.0
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: mypy>=1.10.0; extra == 'dev'
|
|
45
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
46
|
+
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
47
|
+
Description-Content-Type: text/markdown
|
|
48
|
+
|
|
49
|
+
# Decodo tools for CrewAI
|
|
50
|
+
|
|
51
|
+

|
|
52
|
+

|
|
53
|
+
|
|
54
|
+
[CrewAI](https://www.crewai.com) tools for the [Decodo](https://decodo.com) Web Scraping API. Give your agents live web pages and search results.
|
|
55
|
+
|
|
56
|
+
| Tool | What it does |
|
|
57
|
+
| --- | --- |
|
|
58
|
+
| `DecodoWebScraperTool` | Scrapes any public URL and returns the page as Markdown. |
|
|
59
|
+
| `DecodoSearchTool` | Searches Google or Amazon and returns the parsed results as JSON. |
|
|
60
|
+
| `DecodoEcommerceTool` | Gets product details or search results from Amazon, Walmart or Target as JSON. |
|
|
61
|
+
| `DecodoSERPTool` | Gets Google or Bing results pages, with paging, or a Google AI Mode answer as JSON. |
|
|
62
|
+
|
|
63
|
+
## Requirements
|
|
64
|
+
|
|
65
|
+
- Python 3.12 or 3.13
|
|
66
|
+
- A Decodo Web Scraping API key
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
pip install decodo-crewai
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Credentials
|
|
75
|
+
|
|
76
|
+
Copy your API key from your Web Scraping API subscription on the [Decodo dashboard](https://dashboard.decodo.com/web-data/playground) and set it as an environment variable:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
export DECODO_API_KEY="<api_key>"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Or pass it to each tool:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
DecodoWebScraperTool(api_key="<api_key>")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Usage
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from crewai import Agent, Crew, Task
|
|
92
|
+
|
|
93
|
+
from decodo_crewai import DecodoSearchTool, DecodoWebScraperTool
|
|
94
|
+
|
|
95
|
+
researcher = Agent(
|
|
96
|
+
role="Researcher",
|
|
97
|
+
goal="Find and summarize up-to-date information from the web",
|
|
98
|
+
backstory="You search the web, read the most relevant pages and report what they say.",
|
|
99
|
+
tools=[DecodoSearchTool(), DecodoWebScraperTool()],
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
task = Task(
|
|
103
|
+
description="Find the three most recent announcements about {topic} and summarize each one.",
|
|
104
|
+
expected_output="Three bullet points, each with a one-sentence summary and the source URL.",
|
|
105
|
+
agent=researcher,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
crew = Crew(agents=[researcher], tasks=[task])
|
|
109
|
+
print(crew.kickoff(inputs={"topic": "solid-state batteries"}))
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The tools can also be called directly:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from decodo_crewai import DecodoSearchTool, DecodoWebScraperTool
|
|
116
|
+
|
|
117
|
+
print(DecodoWebScraperTool().run(url="https://ip.decodo.com"))
|
|
118
|
+
print(DecodoSearchTool().run(query="mechanical keyboard", source="amazon"))
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
For a complete multi-agent example, see the [research crew](https://github.com/Decodo/crewAI/tree/main/examples/research_crew): a researcher, an analyst and a writer that turn any topic into a cited report.
|
|
122
|
+
|
|
123
|
+
## Tools
|
|
124
|
+
|
|
125
|
+
### DecodoWebScraperTool
|
|
126
|
+
|
|
127
|
+
Arguments the agent fills in:
|
|
128
|
+
|
|
129
|
+
| Argument | Type | Description |
|
|
130
|
+
| --- | --- | --- |
|
|
131
|
+
| `url` | `str` | Full URL of the page to scrape. |
|
|
132
|
+
| `geo` | `str`, optional | Country to send the request from, e.g. `United States`. |
|
|
133
|
+
| `js_render` | `bool`, optional | Open the page in a headless browser. Only needed for pages that load content with JavaScript. |
|
|
134
|
+
|
|
135
|
+
Constructor options:
|
|
136
|
+
|
|
137
|
+
| Option | Default | Description |
|
|
138
|
+
| --- | --- | --- |
|
|
139
|
+
| `api_key` | `DECODO_API_KEY` | Decodo Web Scraping API key. |
|
|
140
|
+
| `max_chars` | `45000` | Markdown longer than this is truncated to keep the page within the agent's context. |
|
|
141
|
+
|
|
142
|
+
### DecodoSearchTool
|
|
143
|
+
|
|
144
|
+
Arguments the agent fills in:
|
|
145
|
+
|
|
146
|
+
| Argument | Type | Description |
|
|
147
|
+
| --- | --- | --- |
|
|
148
|
+
| `query` | `str` | Search query. |
|
|
149
|
+
| `source` | `"google"` or `"amazon"`, optional | `google` (default) for web results, `amazon` for product listings. |
|
|
150
|
+
| `geo` | `str`, optional | Google: a country name, e.g. `United States`. Amazon: a US ZIP code, e.g. `10001`, or a two-letter country code for delivery outside the US, e.g. `DE`. |
|
|
151
|
+
|
|
152
|
+
### DecodoEcommerceTool
|
|
153
|
+
|
|
154
|
+
Arguments the agent fills in. Pass either `product_id` or `query`:
|
|
155
|
+
|
|
156
|
+
| Argument | Type | Description |
|
|
157
|
+
| --- | --- | --- |
|
|
158
|
+
| `store` | `"amazon"`, `"walmart"` or `"target"` | Store to look up. |
|
|
159
|
+
| `product_id` | `str`, optional | One product's details. Amazon: the ASIN, e.g. `B09H74FXNW`. Walmart or Target: the product ID from the product URL. |
|
|
160
|
+
| `query` | `str`, optional | Search the store's products instead. |
|
|
161
|
+
| `zip_code` | `str`, optional | US ZIP code for prices and availability at that delivery location, e.g. `10001`. |
|
|
162
|
+
|
|
163
|
+
### DecodoSERPTool
|
|
164
|
+
|
|
165
|
+
Arguments the agent fills in:
|
|
166
|
+
|
|
167
|
+
| Argument | Type | Description |
|
|
168
|
+
| --- | --- | --- |
|
|
169
|
+
| `query` | `str` | Search query. |
|
|
170
|
+
| `engine` | `"google"`, `"bing"` or `"google_ai_mode"`, optional | `google` (default) or `bing` for a results page; `google_ai_mode` for Google's AI-written answer and the sources it cites. |
|
|
171
|
+
| `page` | `int`, optional | Results page to fetch, starting at 1. Ignored by `google_ai_mode`. |
|
|
172
|
+
| `geo` | `str`, optional | Country to search from, e.g. `United States`. |
|
|
173
|
+
|
|
174
|
+
Bing results are rendered in a headless browser and can take over a minute.
|
|
175
|
+
|
|
176
|
+
## Errors
|
|
177
|
+
|
|
178
|
+
When a request fails, for example because of an invalid API key, a rate limit, a page that returns `404` or a network error that keeps the request from reaching Decodo, the tools return a CrewAI `ToolFailure` instead of raising. The agent sees what went wrong and can retry or change course, and CrewAI records the failure according to the agent's `tool_failure_policy`.
|
|
179
|
+
|
|
180
|
+
## License
|
|
181
|
+
|
|
182
|
+
MIT
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# Decodo tools for CrewAI
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+
|
|
6
|
+
[CrewAI](https://www.crewai.com) tools for the [Decodo](https://decodo.com) Web Scraping API. Give your agents live web pages and search results.
|
|
7
|
+
|
|
8
|
+
| Tool | What it does |
|
|
9
|
+
| --- | --- |
|
|
10
|
+
| `DecodoWebScraperTool` | Scrapes any public URL and returns the page as Markdown. |
|
|
11
|
+
| `DecodoSearchTool` | Searches Google or Amazon and returns the parsed results as JSON. |
|
|
12
|
+
| `DecodoEcommerceTool` | Gets product details or search results from Amazon, Walmart or Target as JSON. |
|
|
13
|
+
| `DecodoSERPTool` | Gets Google or Bing results pages, with paging, or a Google AI Mode answer as JSON. |
|
|
14
|
+
|
|
15
|
+
## Requirements
|
|
16
|
+
|
|
17
|
+
- Python 3.12 or 3.13
|
|
18
|
+
- A Decodo Web Scraping API key
|
|
19
|
+
|
|
20
|
+
## Installation
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
pip install decodo-crewai
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Credentials
|
|
27
|
+
|
|
28
|
+
Copy your API key from your Web Scraping API subscription on the [Decodo dashboard](https://dashboard.decodo.com/web-data/playground) and set it as an environment variable:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
export DECODO_API_KEY="<api_key>"
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or pass it to each tool:
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
DecodoWebScraperTool(api_key="<api_key>")
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from crewai import Agent, Crew, Task
|
|
44
|
+
|
|
45
|
+
from decodo_crewai import DecodoSearchTool, DecodoWebScraperTool
|
|
46
|
+
|
|
47
|
+
researcher = Agent(
|
|
48
|
+
role="Researcher",
|
|
49
|
+
goal="Find and summarize up-to-date information from the web",
|
|
50
|
+
backstory="You search the web, read the most relevant pages and report what they say.",
|
|
51
|
+
tools=[DecodoSearchTool(), DecodoWebScraperTool()],
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
task = Task(
|
|
55
|
+
description="Find the three most recent announcements about {topic} and summarize each one.",
|
|
56
|
+
expected_output="Three bullet points, each with a one-sentence summary and the source URL.",
|
|
57
|
+
agent=researcher,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
crew = Crew(agents=[researcher], tasks=[task])
|
|
61
|
+
print(crew.kickoff(inputs={"topic": "solid-state batteries"}))
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The tools can also be called directly:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from decodo_crewai import DecodoSearchTool, DecodoWebScraperTool
|
|
68
|
+
|
|
69
|
+
print(DecodoWebScraperTool().run(url="https://ip.decodo.com"))
|
|
70
|
+
print(DecodoSearchTool().run(query="mechanical keyboard", source="amazon"))
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
For a complete multi-agent example, see the [research crew](https://github.com/Decodo/crewAI/tree/main/examples/research_crew): a researcher, an analyst and a writer that turn any topic into a cited report.
|
|
74
|
+
|
|
75
|
+
## Tools
|
|
76
|
+
|
|
77
|
+
### DecodoWebScraperTool
|
|
78
|
+
|
|
79
|
+
Arguments the agent fills in:
|
|
80
|
+
|
|
81
|
+
| Argument | Type | Description |
|
|
82
|
+
| --- | --- | --- |
|
|
83
|
+
| `url` | `str` | Full URL of the page to scrape. |
|
|
84
|
+
| `geo` | `str`, optional | Country to send the request from, e.g. `United States`. |
|
|
85
|
+
| `js_render` | `bool`, optional | Open the page in a headless browser. Only needed for pages that load content with JavaScript. |
|
|
86
|
+
|
|
87
|
+
Constructor options:
|
|
88
|
+
|
|
89
|
+
| Option | Default | Description |
|
|
90
|
+
| --- | --- | --- |
|
|
91
|
+
| `api_key` | `DECODO_API_KEY` | Decodo Web Scraping API key. |
|
|
92
|
+
| `max_chars` | `45000` | Markdown longer than this is truncated to keep the page within the agent's context. |
|
|
93
|
+
|
|
94
|
+
### DecodoSearchTool
|
|
95
|
+
|
|
96
|
+
Arguments the agent fills in:
|
|
97
|
+
|
|
98
|
+
| Argument | Type | Description |
|
|
99
|
+
| --- | --- | --- |
|
|
100
|
+
| `query` | `str` | Search query. |
|
|
101
|
+
| `source` | `"google"` or `"amazon"`, optional | `google` (default) for web results, `amazon` for product listings. |
|
|
102
|
+
| `geo` | `str`, optional | Google: a country name, e.g. `United States`. Amazon: a US ZIP code, e.g. `10001`, or a two-letter country code for delivery outside the US, e.g. `DE`. |
|
|
103
|
+
|
|
104
|
+
### DecodoEcommerceTool
|
|
105
|
+
|
|
106
|
+
Arguments the agent fills in. Pass either `product_id` or `query`:
|
|
107
|
+
|
|
108
|
+
| Argument | Type | Description |
|
|
109
|
+
| --- | --- | --- |
|
|
110
|
+
| `store` | `"amazon"`, `"walmart"` or `"target"` | Store to look up. |
|
|
111
|
+
| `product_id` | `str`, optional | One product's details. Amazon: the ASIN, e.g. `B09H74FXNW`. Walmart or Target: the product ID from the product URL. |
|
|
112
|
+
| `query` | `str`, optional | Search the store's products instead. |
|
|
113
|
+
| `zip_code` | `str`, optional | US ZIP code for prices and availability at that delivery location, e.g. `10001`. |
|
|
114
|
+
|
|
115
|
+
### DecodoSERPTool
|
|
116
|
+
|
|
117
|
+
Arguments the agent fills in:
|
|
118
|
+
|
|
119
|
+
| Argument | Type | Description |
|
|
120
|
+
| --- | --- | --- |
|
|
121
|
+
| `query` | `str` | Search query. |
|
|
122
|
+
| `engine` | `"google"`, `"bing"` or `"google_ai_mode"`, optional | `google` (default) or `bing` for a results page; `google_ai_mode` for Google's AI-written answer and the sources it cites. |
|
|
123
|
+
| `page` | `int`, optional | Results page to fetch, starting at 1. Ignored by `google_ai_mode`. |
|
|
124
|
+
| `geo` | `str`, optional | Country to search from, e.g. `United States`. |
|
|
125
|
+
|
|
126
|
+
Bing results are rendered in a headless browser and can take over a minute.
|
|
127
|
+
|
|
128
|
+
## Errors
|
|
129
|
+
|
|
130
|
+
When a request fails, for example because of an invalid API key, a rate limit, a page that returns `404` or a network error that keeps the request from reaching Decodo, the tools return a CrewAI `ToolFailure` instead of raising. The agent sees what went wrong and can retry or change course, and CrewAI records the failure according to the agent's `tool_failure_policy`.
|
|
131
|
+
|
|
132
|
+
## License
|
|
133
|
+
|
|
134
|
+
MIT
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# Research crew
|
|
2
|
+
|
|
3
|
+
A three-agent [CrewAI](https://www.crewai.com) crew that researches any topic on the live web with Decodo and writes a cited report.
|
|
4
|
+
|
|
5
|
+
| Agent | Tool | Job |
|
|
6
|
+
| --- | --- | --- |
|
|
7
|
+
| Web researcher | `DecodoSearchTool` | Searches Google and picks the 5 most relevant sources. |
|
|
8
|
+
| Research analyst | `DecodoWebScraperTool` | Reads each source and extracts the key facts. |
|
|
9
|
+
| Report writer | none | Turns the notes into a Markdown report with citations. |
|
|
10
|
+
|
|
11
|
+
The agents run in sequence, each one working from the previous one's output. The final report is printed and saved to `report.md`.
|
|
12
|
+
|
|
13
|
+
## Setup
|
|
14
|
+
|
|
15
|
+
From the repository root:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
python3.12 -m venv .venv
|
|
19
|
+
source .venv/bin/activate
|
|
20
|
+
pip install -e .
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Set your keys in a `.env` file, in this folder or the repository root, or export them in your shell:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
DECODO_API_KEY=<decodo_api_key>
|
|
27
|
+
OPENAI_API_KEY=<openai_api_key>
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Get the Decodo API key from your Web Scraping API subscription on the [Decodo dashboard](https://dashboard.decodo.com/web-data/playground).
|
|
31
|
+
|
|
32
|
+
The crew uses CrewAI's default model, `gpt-4.1-mini`. To use another model, set `MODEL`, e.g. `MODEL=gpt-4.1`. Other providers need their CrewAI extra and key, e.g. `pip install "crewai[anthropic]"`, `ANTHROPIC_API_KEY` and `MODEL=anthropic/<model>`.
|
|
33
|
+
|
|
34
|
+
## Run
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
cd examples/research_crew
|
|
38
|
+
python main.py "heat pumps in cold climates"
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Without a topic, it researches "the state of solid-state batteries for electric cars".
|
|
42
|
+
|
|
43
|
+
## Sample output
|
|
44
|
+
|
|
45
|
+
`python main.py` with the default topic and `gpt-4.1-mini` made 1 search and 5 page scrapes and wrote this report (October 2026):
|
|
46
|
+
|
|
47
|
+
<details>
|
|
48
|
+
<summary>report.md</summary>
|
|
49
|
+
|
|
50
|
+
# The State of Solid-State Batteries for Electric Cars
|
|
51
|
+
|
|
52
|
+
Solid-state batteries (SSBs) represent a promising evolution in electric vehicle (EV) energy storage technology, offering higher energy density, improved safety, and longer lifespan compared to conventional lithium-ion batteries. While significant technical progress has been made and multiple automakers target commercialization around 2027-2030, challenges in manufacturing scale-up and cost reduction mean widespread adoption remains several years away.
|
|
53
|
+
|
|
54
|
+
## Energy Density and Performance Advantages
|
|
55
|
+
|
|
56
|
+
SSBs replace the liquid electrolyte in lithium-ion batteries with a solid electrolyte, enabling theoretical energy densities exceeding 500 Wh/kg, compared to 250-300 Wh/kg typical of current lithium-ion batteries. This can translate into EV ranges of 600 to 750 miles and notably faster charging times, such as 10-minute fast charging targets set by Toyota and Samsung SDI for their all-solid-state battery (ASSB) development programs [1][2][4]. Safety is also improved due to the non-flammable nature of solid electrolytes and enhanced chemical stability [4][5].
|
|
57
|
+
|
|
58
|
+
## Industry Development and Timelines
|
|
59
|
+
|
|
60
|
+
Automakers and battery companies including Toyota, Samsung SDI, Volkswagen, QuantumScape, Honda, and Factorial Energy are actively developing various solid-state and semi-solid-state battery technologies. Semi-solid-state batteries, offering an intermediate step with improved energy density and reduced risk, have already been produced in limited forms—such as the Nio WELION 150-kWh pack with 260 Wh/kg in 2024 [2].
|
|
61
|
+
|
|
62
|
+
Full mass production of all-solid-state batteries is broadly expected to begin around 2027-2030, initially targeting premium and luxury vehicle segments. Toyota and Samsung SDI have pilot production lines aimed at 2027, while Honda plans demonstration production lines in 2024. Factorial Energy has demonstrated semi-solid-state packs with exceptional range [2][4][5]. However, industry insiders caution that large-scale commercial production may extend beyond 2030 to achieve cost-competitiveness and manufacturing reliability [1][3].
|
|
63
|
+
|
|
64
|
+
## Manufacturing and Cost Challenges
|
|
65
|
+
|
|
66
|
+
Despite promising performance, manufacturing SSBs at scale remains a major barrier. The production process requires advanced techniques to ensure dense, defect-free solid electrolyte and electrode interfaces, such as stamping and roll-pressing, which are still being refined [5]. Additional challenges include dendrite formation, raw material costs, and establishing consistent quality control [1][4].
|
|
67
|
+
|
|
68
|
+
Cost projections vary: optimistic scenarios predict $140 per kWh by 2028 with mass production, whereas pessimistic views suggest delays and prices near $175 per kWh around 2032-2033 [1]. These higher costs currently limit initial applications to premium vehicles, with broader market penetration likely delayed until mid-2030s [3].
|
|
69
|
+
|
|
70
|
+
## Market Impact and Outlook
|
|
71
|
+
|
|
72
|
+
While SSBs have the potential to revolutionize EV range, charging speed, and safety, incremental improvements in existing lithium-ion battery technologies—such as LFP cells with already-fast charging and 300-mile ranges—moderate the urgency to transition immediately [3]. The market outlook thus anticipates a gradual integration, with lithium-ion batteries evolving in parallel as SSB technology matures.
|
|
73
|
+
|
|
74
|
+
In summary, solid-state batteries are poised to become a critical technology milestone for EVs within this decade. Their successful commercialization depends on overcoming significant manufacturing and cost hurdles. Expected first adoption is in high-end vehicles by 2027-2030, followed by wider penetration in the 2030s.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Sources
|
|
79
|
+
|
|
80
|
+
[1] Solid-state batteries, their future in the energy storage and electric vehicles market
|
|
81
|
+
https://www.sciencedirect.com/science/article/pii/S2772569324000902
|
|
82
|
+
|
|
83
|
+
[2] We Keep Hearing About Solid-State Batteries. Here's What'll Actually Reach the Road.
|
|
84
|
+
https://www.motortrend.com/features/solid-state-batteries-future-cars
|
|
85
|
+
|
|
86
|
+
[3] Will Solid-state batteries be the EV game changer when they start appearing in 2027/2028? (Reddit r/electricvehicles discussion)
|
|
87
|
+
https://www.reddit.com/r/electricvehicles/comments/1vz26my/will_solidstate_batteries_be_the_ev_game_changer/
|
|
88
|
+
|
|
89
|
+
[4] Solid-State Battery News: Samsung & Toyota
|
|
90
|
+
https://www.monolithai.com/blog/solid-state-battery-news
|
|
91
|
+
|
|
92
|
+
[5] All-solid-state battery technology (Honda official page)
|
|
93
|
+
https://global.honda/en/tech/All-solid-state_battery_technology/
|
|
94
|
+
|
|
95
|
+
</details>
|