llama-index-tools-contextfetch 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llama_index_tools_contextfetch-0.1.0/.gitignore +20 -0
- llama_index_tools_contextfetch-0.1.0/LICENSE +21 -0
- llama_index_tools_contextfetch-0.1.0/PKG-INFO +65 -0
- llama_index_tools_contextfetch-0.1.0/README.md +49 -0
- llama_index_tools_contextfetch-0.1.0/llama_index/tools/contextfetch/__init__.py +3 -0
- llama_index_tools_contextfetch-0.1.0/llama_index/tools/contextfetch/base.py +119 -0
- llama_index_tools_contextfetch-0.1.0/pyproject.toml +73 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
node_modules/
|
|
3
|
+
.next/
|
|
4
|
+
dist/
|
|
5
|
+
engine/.model_cache/
|
|
6
|
+
engine/.data/
|
|
7
|
+
.env
|
|
8
|
+
.env.npm
|
|
9
|
+
.env.webshare
|
|
10
|
+
.venv
|
|
11
|
+
engine/.model_cache
|
|
12
|
+
growth-sniper/leads.csv
|
|
13
|
+
__pycache__/
|
|
14
|
+
.pytest_cache/
|
|
15
|
+
.env.hetzner
|
|
16
|
+
.next-*/
|
|
17
|
+
.vercel/
|
|
18
|
+
benchmarks/data/
|
|
19
|
+
*.tsbuildinfo
|
|
20
|
+
next-env.d.ts
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 ContextFetch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: llama-index-tools-contextfetch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: llama-index tools contextfetch integration
|
|
5
|
+
Project-URL: Homepage, https://contextfetch.dev
|
|
6
|
+
Project-URL: Documentation, https://contextfetch.dev/docs/reference
|
|
7
|
+
Author: ContextFetch
|
|
8
|
+
Maintainer: lachy3433
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: contextfetch,markdown,web,web search
|
|
12
|
+
Requires-Python: <4.0,>=3.10
|
|
13
|
+
Requires-Dist: contextfetch>=0.1.0
|
|
14
|
+
Requires-Dist: llama-index-core<0.15,>=0.13.0
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# LlamaIndex Tools Integration: ContextFetch
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install llama-index-tools-contextfetch
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
This tool connects agents to [ContextFetch](https://contextfetch.dev), a web
|
|
24
|
+
search and web page to Markdown API.
|
|
25
|
+
|
|
26
|
+
Pass `api_key`, or set the `CONTEXTFETCH_API_KEY` environment variable. With
|
|
27
|
+
neither, requests use ContextFetch's keyless tier, a small daily allowance per IP.
|
|
28
|
+
The `contextfetch` package can request a free starter key:
|
|
29
|
+
`from contextfetch import ContextFetch; ContextFetch().free_key()`.
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
# %pip install llama-index llama-index-tools-contextfetch
|
|
35
|
+
|
|
36
|
+
from llama_index.tools.contextfetch import ContextFetchToolSpec
|
|
37
|
+
from llama_index.core.agent.workflow import FunctionAgent
|
|
38
|
+
from llama_index.llms.openai import OpenAI
|
|
39
|
+
|
|
40
|
+
contextfetch_tool = ContextFetchToolSpec() # or ContextFetchToolSpec(api_key="...")
|
|
41
|
+
agent = FunctionAgent(
|
|
42
|
+
tools=contextfetch_tool.to_tool_list(),
|
|
43
|
+
llm=OpenAI(model="gpt-4.1"),
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
print(await agent.run("How do asyncio task groups handle errors?"))
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Available Functions
|
|
50
|
+
|
|
51
|
+
`search`: Search the web. Returns each result's title, URL, published date when
|
|
52
|
+
known, and the passages relevant to the query.
|
|
53
|
+
|
|
54
|
+
`read_page`: Read a web page as Markdown. With `query`, only the parts of the page
|
|
55
|
+
relevant to it are returned.
|
|
56
|
+
|
|
57
|
+
`find_similar`: Find pages on other sites that are similar to a given page.
|
|
58
|
+
|
|
59
|
+
`answer`: Answer a question with passages quoted verbatim from web pages, each
|
|
60
|
+
followed by `[n]`, then the numbered source URLs. No text is generated.
|
|
61
|
+
|
|
62
|
+
Each function returns a Markdown string that names its sources by title and URL.
|
|
63
|
+
To load pages as Documents for indexing, use `llama-index-readers-contextfetch`.
|
|
64
|
+
|
|
65
|
+
This loader is designed to be used as a way to load data as a Tool in an Agent.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# LlamaIndex Tools Integration: ContextFetch
|
|
2
|
+
|
|
3
|
+
```bash
|
|
4
|
+
pip install llama-index-tools-contextfetch
|
|
5
|
+
```
|
|
6
|
+
|
|
7
|
+
This tool connects agents to [ContextFetch](https://contextfetch.dev), a web
|
|
8
|
+
search and web page to Markdown API.
|
|
9
|
+
|
|
10
|
+
Pass `api_key`, or set the `CONTEXTFETCH_API_KEY` environment variable. With
|
|
11
|
+
neither, requests use ContextFetch's keyless tier, a small daily allowance per IP.
|
|
12
|
+
The `contextfetch` package can request a free starter key:
|
|
13
|
+
`from contextfetch import ContextFetch; ContextFetch().free_key()`.
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
# %pip install llama-index llama-index-tools-contextfetch
|
|
19
|
+
|
|
20
|
+
from llama_index.tools.contextfetch import ContextFetchToolSpec
|
|
21
|
+
from llama_index.core.agent.workflow import FunctionAgent
|
|
22
|
+
from llama_index.llms.openai import OpenAI
|
|
23
|
+
|
|
24
|
+
contextfetch_tool = ContextFetchToolSpec() # or ContextFetchToolSpec(api_key="...")
|
|
25
|
+
agent = FunctionAgent(
|
|
26
|
+
tools=contextfetch_tool.to_tool_list(),
|
|
27
|
+
llm=OpenAI(model="gpt-4.1"),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
print(await agent.run("How do asyncio task groups handle errors?"))
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Available Functions
|
|
34
|
+
|
|
35
|
+
`search`: Search the web. Returns each result's title, URL, published date when
|
|
36
|
+
known, and the passages relevant to the query.
|
|
37
|
+
|
|
38
|
+
`read_page`: Read a web page as Markdown. With `query`, only the parts of the page
|
|
39
|
+
relevant to it are returned.
|
|
40
|
+
|
|
41
|
+
`find_similar`: Find pages on other sites that are similar to a given page.
|
|
42
|
+
|
|
43
|
+
`answer`: Answer a question with passages quoted verbatim from web pages, each
|
|
44
|
+
followed by `[n]`, then the numbered source URLs. No text is generated.
|
|
45
|
+
|
|
46
|
+
Each function returns a Markdown string that names its sources by title and URL.
|
|
47
|
+
To load pages as Documents for indexing, use `llama-index-readers-contextfetch`.
|
|
48
|
+
|
|
49
|
+
This loader is designed to be used as a way to load data as a Tool in an Agent.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""ContextFetch tool spec."""
|
|
2
|
+
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
|
|
5
|
+
from contextfetch import ContextFetch, Result
|
|
6
|
+
from llama_index.core.tools.tool_spec.base import BaseToolSpec
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ContextFetchToolSpec(BaseToolSpec):
|
|
10
|
+
"""
|
|
11
|
+
ContextFetch tool spec: web search and web pages as Markdown.
|
|
12
|
+
|
|
13
|
+
Each function returns a Markdown string that names its sources by title
|
|
14
|
+
and URL, so the agent can cite them or read them in full.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
api_key: ContextFetch API key. Defaults to the CONTEXTFETCH_API_KEY
|
|
18
|
+
environment variable; with neither, requests use the keyless tier.
|
|
19
|
+
|
|
20
|
+
Example:
|
|
21
|
+
.. code-block:: python
|
|
22
|
+
|
|
23
|
+
from llama_index.tools.contextfetch import ContextFetchToolSpec
|
|
24
|
+
from llama_index.core.agent.workflow import FunctionAgent
|
|
25
|
+
from llama_index.llms.openai import OpenAI
|
|
26
|
+
|
|
27
|
+
agent = FunctionAgent(
|
|
28
|
+
tools=ContextFetchToolSpec().to_tool_list(),
|
|
29
|
+
llm=OpenAI(model="gpt-4.1"),
|
|
30
|
+
)
|
|
31
|
+
print(await agent.run("How do asyncio task groups handle errors?"))
|
|
32
|
+
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
spec_functions = ["search", "read_page", "find_similar", "answer"]
|
|
36
|
+
|
|
37
|
+
def __init__(self, api_key: Optional[str] = None) -> None:
|
|
38
|
+
"""Initialize with parameters."""
|
|
39
|
+
self.client = ContextFetch(api_key=api_key)
|
|
40
|
+
|
|
41
|
+
def search(
|
|
42
|
+
self,
|
|
43
|
+
query: str,
|
|
44
|
+
num_results: int = 5,
|
|
45
|
+
include_domains: Optional[List[str]] = None,
|
|
46
|
+
) -> str:
|
|
47
|
+
"""
|
|
48
|
+
Search the web. Returns each result's title, URL and the passages relevant to the query.
|
|
49
|
+
|
|
50
|
+
Use read_page to get a result's full text.
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
query: What to search for, in natural language.
|
|
54
|
+
num_results: How many results to return, 1 to 25.
|
|
55
|
+
include_domains: Only return results from these domains, e.g. ["docs.python.org"].
|
|
56
|
+
|
|
57
|
+
"""
|
|
58
|
+
response = self.client.search(
|
|
59
|
+
query,
|
|
60
|
+
num_results=num_results,
|
|
61
|
+
include_domains=include_domains,
|
|
62
|
+
pruned_context=True,
|
|
63
|
+
)
|
|
64
|
+
return _format_results(response.get("results", []))
|
|
65
|
+
|
|
66
|
+
def read_page(self, url: str, query: Optional[str] = None) -> str:
|
|
67
|
+
"""
|
|
68
|
+
Read a web page as Markdown.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
url: The page to read.
|
|
72
|
+
query: Optional question. When given, only the parts of the page relevant to it are returned.
|
|
73
|
+
|
|
74
|
+
"""
|
|
75
|
+
return self.client.read(url, query=query).markdown
|
|
76
|
+
|
|
77
|
+
def find_similar(self, url: str, num_results: int = 5) -> str:
|
|
78
|
+
"""
|
|
79
|
+
Find pages on other sites that are similar to a web page. Returns each result's title, URL and its most relevant passages.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
url: The page to find similar pages for.
|
|
83
|
+
num_results: How many results to return, 1 to 25.
|
|
84
|
+
|
|
85
|
+
"""
|
|
86
|
+
response = self.client.find_similar(
|
|
87
|
+
url, num_results=num_results, pruned_context=True
|
|
88
|
+
)
|
|
89
|
+
return _format_results(response.get("results", []))
|
|
90
|
+
|
|
91
|
+
def answer(self, query: str) -> str:
|
|
92
|
+
"""
|
|
93
|
+
Answer a question with passages quoted verbatim from web pages, each followed by [n], then the numbered sources.
|
|
94
|
+
|
|
95
|
+
No text is generated; when no passage is relevant enough, says so.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
query: The question.
|
|
99
|
+
|
|
100
|
+
"""
|
|
101
|
+
response = self.client.answer(query)
|
|
102
|
+
if not response.get("answer"):
|
|
103
|
+
return "No passage was relevant enough to answer this question."
|
|
104
|
+
sources = "\n".join(
|
|
105
|
+
f"[{n}] {c.get('title') or c.get('url')}: {c.get('url')}"
|
|
106
|
+
for n, c in enumerate(response.get("citations", []), start=1)
|
|
107
|
+
)
|
|
108
|
+
return f"{response['answer']}\n\nSources:\n{sources}"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _format_results(results: List[Result]) -> str:
|
|
112
|
+
blocks = []
|
|
113
|
+
for r in results:
|
|
114
|
+
lines = [f"## {r.get('title') or r.get('url')}", f"URL: {r.get('url')}"]
|
|
115
|
+
if r.get("publishedDate"):
|
|
116
|
+
lines.append(f"Published: {r['publishedDate']}")
|
|
117
|
+
lines += ["", r.get("prunedContext") or ""]
|
|
118
|
+
blocks.append("\n".join(lines).strip())
|
|
119
|
+
return "\n\n".join(blocks) or "No results."
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[dependency-groups]
|
|
6
|
+
dev = [
|
|
7
|
+
"black[jupyter]>=23.7.0,<=26.5.1",
|
|
8
|
+
"codespell[toml]>=v2.2.6",
|
|
9
|
+
"diff-cover>=9.2.0",
|
|
10
|
+
"ipython==8.10.0",
|
|
11
|
+
"jupyter>=1.0.0,<2",
|
|
12
|
+
"mypy==0.991",
|
|
13
|
+
"pre-commit==3.2.0",
|
|
14
|
+
"pylint==2.15.10",
|
|
15
|
+
"pytest",
|
|
16
|
+
"pytest-cov>=6.1.1",
|
|
17
|
+
"pytest-mock==3.11.1",
|
|
18
|
+
"ruff==0.11.11",
|
|
19
|
+
"types-Deprecated>=0.1.0",
|
|
20
|
+
"types-PyYAML>=6.0.12.12,<7",
|
|
21
|
+
"types-protobuf>=4.24.0.4,<5",
|
|
22
|
+
"types-redis==4.5.5.0",
|
|
23
|
+
"types-requests==2.28.11.8",
|
|
24
|
+
"types-setuptools==67.1.0.0",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project]
|
|
28
|
+
name = "llama-index-tools-contextfetch"
|
|
29
|
+
version = "0.1.0"
|
|
30
|
+
description = "llama-index tools contextfetch integration"
|
|
31
|
+
authors = [{name = "ContextFetch"}]
|
|
32
|
+
requires-python = ">=3.10,<4.0"
|
|
33
|
+
readme = "README.md"
|
|
34
|
+
license = "MIT"
|
|
35
|
+
maintainers = [{name = "lachy3433"}]
|
|
36
|
+
keywords = [
|
|
37
|
+
"contextfetch",
|
|
38
|
+
"markdown",
|
|
39
|
+
"web",
|
|
40
|
+
"web search",
|
|
41
|
+
]
|
|
42
|
+
dependencies = [
|
|
43
|
+
"contextfetch>=0.1.0",
|
|
44
|
+
"llama-index-core>=0.13.0,<0.15",
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
[project.urls]
|
|
48
|
+
Homepage = "https://contextfetch.dev"
|
|
49
|
+
Documentation = "https://contextfetch.dev/docs/reference"
|
|
50
|
+
|
|
51
|
+
[tool.codespell]
|
|
52
|
+
check-filenames = true
|
|
53
|
+
check-hidden = true
|
|
54
|
+
skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
|
|
55
|
+
|
|
56
|
+
[tool.hatch.build.targets.sdist]
|
|
57
|
+
include = ["llama_index/"]
|
|
58
|
+
|
|
59
|
+
[tool.hatch.build.targets.wheel]
|
|
60
|
+
include = ["llama_index/"]
|
|
61
|
+
|
|
62
|
+
[tool.llamahub]
|
|
63
|
+
contains_example = false
|
|
64
|
+
import_path = "llama_index.tools.contextfetch"
|
|
65
|
+
|
|
66
|
+
[tool.llamahub.class_authors]
|
|
67
|
+
ContextFetchToolSpec = "lachy3433"
|
|
68
|
+
|
|
69
|
+
[tool.mypy]
|
|
70
|
+
disallow_untyped_defs = true
|
|
71
|
+
exclude = ["_static", "build", "examples", "notebooks", "venv"]
|
|
72
|
+
ignore_missing_imports = true
|
|
73
|
+
python_version = "3.8"
|