langchain-diffbot 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- langchain_diffbot-0.1.0/.gitignore +16 -0
- langchain_diffbot-0.1.0/LICENSE +21 -0
- langchain_diffbot-0.1.0/PKG-INFO +255 -0
- langchain_diffbot-0.1.0/README.md +223 -0
- langchain_diffbot-0.1.0/langchain_diffbot/__init__.py +39 -0
- langchain_diffbot-0.1.0/langchain_diffbot/_base.py +110 -0
- langchain_diffbot-0.1.0/langchain_diffbot/chat_models.py +122 -0
- langchain_diffbot-0.1.0/langchain_diffbot/document_loaders.py +144 -0
- langchain_diffbot-0.1.0/langchain_diffbot/py.typed +0 -0
- langchain_diffbot-0.1.0/langchain_diffbot/retrievers.py +297 -0
- langchain_diffbot-0.1.0/langchain_diffbot/tools.py +435 -0
- langchain_diffbot-0.1.0/pyproject.toml +97 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Diffbot
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: langchain-diffbot
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: LangChain integration for the Diffbot Knowledge Graph and Extract APIs
|
|
5
|
+
Project-URL: Homepage, https://www.diffbot.com/
|
|
6
|
+
Project-URL: Repository, https://github.com/diffbot/langchain-diffbot
|
|
7
|
+
Project-URL: Issues, https://github.com/diffbot/langchain-diffbot/issues
|
|
8
|
+
Author: Diffbot
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: <4.0,>=3.10
|
|
21
|
+
Requires-Dist: diffbot-python>=0.2.1
|
|
22
|
+
Requires-Dist: httpx<1.0,>=0.27
|
|
23
|
+
Requires-Dist: langchain-core<2.0,>=1.0
|
|
24
|
+
Provides-Extra: examples
|
|
25
|
+
Requires-Dist: fastapi<1.0,>=0.115; extra == 'examples'
|
|
26
|
+
Requires-Dist: langchain-anthropic<2.0,>=1.4; extra == 'examples'
|
|
27
|
+
Requires-Dist: langchain<2.0,>=1.3; extra == 'examples'
|
|
28
|
+
Requires-Dist: langsmith<1.0,>=0.1; extra == 'examples'
|
|
29
|
+
Requires-Dist: python-dotenv<2.0,>=1.0; extra == 'examples'
|
|
30
|
+
Requires-Dist: uvicorn[standard]<1.0,>=0.30; extra == 'examples'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# langchain-diffbot
|
|
34
|
+
|
|
35
|
+
A thin LangChain integration over the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. Every Diffbot API gets the closest LangChain primitive:
|
|
36
|
+
|
|
37
|
+
| Diffbot API | LangChain class(es) |
|
|
38
|
+
| --- | --- |
|
|
39
|
+
| Knowledge Graph (DQL) | `DiffbotKnowledgeGraphRetriever`, `DiffbotKnowledgeGraphTool` |
|
|
40
|
+
| Web Search | `DiffbotWebSearchRetriever`, `DiffbotWebSearchTool` |
|
|
41
|
+
| Extract (Analyze) | `DiffbotExtractTool`, `DiffbotExtractLoader` |
|
|
42
|
+
| NLP entities | `DiffbotEntitiesTool` |
|
|
43
|
+
| Crawl | `DiffbotCrawlLoader` |
|
|
44
|
+
| LLM RAG (`ask`) | `ChatDiffbot` (with native streaming) |
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install langchain-diffbot
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Authentication
|
|
53
|
+
|
|
54
|
+
Get an API token at https://app.diffbot.com/get-started/ and export it:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
export DIFFBOT_API_TOKEN="..."
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Every class also accepts `diffbot_api_token=...` directly, or a pre-built `diffbot.Diffbot` client via `client=...` (see [Bring-your-own-client](#bring-your-own-client) below).
|
|
61
|
+
|
|
62
|
+
## Quickstart — Knowledge Graph retriever
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
66
|
+
|
|
67
|
+
retriever = DiffbotKnowledgeGraphRetriever(k=5)
|
|
68
|
+
docs = retriever.invoke("type:Organization industries:\"Artificial Intelligence\" location.city.name:\"Boston\"")
|
|
69
|
+
for d in docs:
|
|
70
|
+
print(d.metadata["name"], "—", d.page_content[:120])
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The query string is a [DQL (Diffbot Query Language)](https://docs.diffbot.com/reference/dql-quickstart) expression.
|
|
74
|
+
|
|
75
|
+
## Shaping the output
|
|
76
|
+
|
|
77
|
+
Diffbot KG entities and web-search results are large. Dumping them straight into an LLM prompt can blow past per-minute input-token limits in a single call. Both retrievers expose three shaping knobs:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from langchain_core.documents import Document
|
|
81
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
82
|
+
|
|
83
|
+
# 1. Project only the top-level fields you care about. Drops everything else
|
|
84
|
+
# from `metadata`. Recommended for agent / tool-use scenarios.
|
|
85
|
+
retriever = DiffbotKnowledgeGraphRetriever(
|
|
86
|
+
k=5,
|
|
87
|
+
fields=["id", "type", "name", "homepageUri", "nbEmployees"],
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# 2. Choose which field becomes `page_content`. First non-empty value wins.
|
|
91
|
+
retriever = DiffbotKnowledgeGraphRetriever(
|
|
92
|
+
content_fields=["summary", "description", "name"],
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# 3. For total control, pass a `document_mapper` that turns a raw entity
|
|
96
|
+
# dict into whatever Document shape you want.
|
|
97
|
+
def mapper(entity: dict) -> Document:
|
|
98
|
+
return Document(
|
|
99
|
+
page_content=entity.get("summary", ""),
|
|
100
|
+
metadata={"id": entity["id"], "name": entity["name"]},
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
retriever = DiffbotKnowledgeGraphRetriever(document_mapper=mapper)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`fields` and `content_fields` are ignored when `document_mapper` is set. The same knobs work on `DiffbotWebSearchRetriever`.
|
|
107
|
+
|
|
108
|
+
## Web search
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from langchain_diffbot import DiffbotWebSearchRetriever
|
|
112
|
+
|
|
113
|
+
web = DiffbotWebSearchRetriever(k=5, fields=["title", "pageUrl", "score"])
|
|
114
|
+
docs = web.invoke("diffbot knowledge graph llm grounding")
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Extract a URL
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from langchain_diffbot import DiffbotExtractTool, DiffbotExtractLoader
|
|
121
|
+
|
|
122
|
+
# Single URL
|
|
123
|
+
tool = DiffbotExtractTool()
|
|
124
|
+
page = tool.invoke({"url": "https://www.diffbot.com/products/extract/"})
|
|
125
|
+
|
|
126
|
+
# Batch — yields one Document per URL, sync or async
|
|
127
|
+
loader = DiffbotExtractLoader(urls=["https://example.com", "https://diffbot.com"])
|
|
128
|
+
for doc in loader.lazy_load():
|
|
129
|
+
print(doc.metadata["title"], doc.page_content[:200])
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`.
|
|
133
|
+
|
|
134
|
+
## ChatDiffbot
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
from langchain_core.messages import HumanMessage
|
|
138
|
+
from langchain_diffbot import ChatDiffbot
|
|
139
|
+
|
|
140
|
+
llm = ChatDiffbot()
|
|
141
|
+
|
|
142
|
+
for chunk in llm.stream([HumanMessage(content="What is the Diffbot Knowledge Graph?")]):
|
|
143
|
+
print(chunk.content, end="", flush=True)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
`_stream` / `_astream` are native — no thread-pool fallback. `.invoke()` aggregates the stream into a single message.
|
|
147
|
+
|
|
148
|
+
## Using a retriever in a chain
|
|
149
|
+
|
|
150
|
+
The retrievers are standard `BaseRetriever`s, so they slot into LCEL like any other:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from langchain_anthropic import ChatAnthropic
|
|
154
|
+
from langchain_core.output_parsers import StrOutputParser
|
|
155
|
+
from langchain_core.prompts import ChatPromptTemplate
|
|
156
|
+
from langchain_core.runnables import RunnablePassthrough
|
|
157
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
158
|
+
|
|
159
|
+
retriever = DiffbotKnowledgeGraphRetriever(
|
|
160
|
+
k=5,
|
|
161
|
+
fields=["id", "name", "homepageUri", "nbEmployees", "industries"],
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
prompt = ChatPromptTemplate.from_template(
|
|
165
|
+
"Answer using only this Diffbot KG context:\n\n{context}\n\nQuestion: {question}"
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _format(docs):
|
|
170
|
+
return "\n---\n".join(
|
|
171
|
+
f"{d.metadata.get('name')} (id={d.metadata.get('id')}): {d.page_content}"
|
|
172
|
+
for d in docs
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
chain = (
|
|
177
|
+
{"context": retriever | _format, "question": RunnablePassthrough()}
|
|
178
|
+
| prompt
|
|
179
|
+
| ChatAnthropic(model="claude-sonnet-4-6")
|
|
180
|
+
| StrOutputParser()
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
chain.invoke('type:Organization location.city.name:"Boston" industries:"Biotech"')
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Bring-your-own-client
|
|
187
|
+
|
|
188
|
+
Every class accepts a pre-built `diffbot.Diffbot` (or `diffbot.DiffbotAsync`) via `client` / `async_client`. The package uses it as-is and **does not close it** — you own the lifecycle. This is the escape hatch for anything the SDK supports that's not re-exposed as a field (custom URLs, `transport=`, shared connection pools, custom headers).
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
from diffbot import Diffbot
|
|
192
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
193
|
+
|
|
194
|
+
# One client shared across many retriever calls (no per-call httpx pool churn)
|
|
195
|
+
shared = Diffbot(token="...", timeout=60.0)
|
|
196
|
+
retriever = DiffbotKnowledgeGraphRetriever(client=shared, k=5)
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
## Examples
|
|
200
|
+
|
|
201
|
+
The [`examples/`](./examples) folder has runnable demos:
|
|
202
|
+
|
|
203
|
+
- [`examples/quickstart/`](./examples/quickstart) — full tour: every public class, output shaping, async, and a multi-tool research agent.
|
|
204
|
+
- [`examples/company_research/`](./examples/company_research) — the same multi-tool agent as a one-shot CLI: `cd examples && python -m company_research "your question"`. The agent combines KG search + web search + URL extract.
|
|
205
|
+
|
|
206
|
+
Both need `langchain` + `langchain-anthropic` on top of the base package — install the extra:
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
pip install "langchain-diffbot[examples]"
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## Development
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
uv sync --all-groups
|
|
216
|
+
uv run pytest tests/unit_tests
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
## Releasing
|
|
220
|
+
|
|
221
|
+
Tokens are stored in macOS Keychain so the Makefile can pull them automatically — no plaintext on disk, no shell-history leaks. First-time setup:
|
|
222
|
+
|
|
223
|
+
```bash
|
|
224
|
+
make set-token-testpypi # prompts; input is hidden as you paste
|
|
225
|
+
make set-token-pypi # same, for real PyPI
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Both targets read with `bash read -rsp` (hidden input), overwrite any existing entry, and never put the token in `make` output or shell history. Re-run either one any time you rotate a token.
|
|
229
|
+
|
|
230
|
+
Then the release flow per version:
|
|
231
|
+
|
|
232
|
+
```bash
|
|
233
|
+
# 1. Bump the version (edits pyproject.toml in place via `uv version --bump`)
|
|
234
|
+
make bump-patch # 0.1.0 → 0.1.1
|
|
235
|
+
# or: make bump-minor # 0.1.0 → 0.2.0
|
|
236
|
+
# or: make bump-major # 0.1.0 → 1.0.0
|
|
237
|
+
|
|
238
|
+
# 2. Publish to TestPyPI and verify installable
|
|
239
|
+
make release-test
|
|
240
|
+
make verify-release-test
|
|
241
|
+
|
|
242
|
+
# 3. Publish to real PyPI (prompts for the version to confirm)
|
|
243
|
+
make release
|
|
244
|
+
make verify-release
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
`make release-test` and `make release` will refuse to publish if the current `pyproject.toml` version is already on the target index — so the workflow is "bump → release-test → verify → release → verify".
|
|
248
|
+
|
|
249
|
+
To rotate: revoke the old token at https://pypi.org/manage/account/token/ (or the TestPyPI equivalent), then run `make set-token-pypi` / `make set-token-testpypi` again — it overwrites the existing Keychain entry without prompting.
|
|
250
|
+
|
|
251
|
+
Integration tests hit the live Diffbot API and require `DIFFBOT_API_TOKEN`:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
uv run pytest tests/integration_tests
|
|
255
|
+
```
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
# langchain-diffbot
|
|
2
|
+
|
|
3
|
+
A thin LangChain integration over the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. Every Diffbot API gets the closest LangChain primitive:
|
|
4
|
+
|
|
5
|
+
| Diffbot API | LangChain class(es) |
|
|
6
|
+
| --- | --- |
|
|
7
|
+
| Knowledge Graph (DQL) | `DiffbotKnowledgeGraphRetriever`, `DiffbotKnowledgeGraphTool` |
|
|
8
|
+
| Web Search | `DiffbotWebSearchRetriever`, `DiffbotWebSearchTool` |
|
|
9
|
+
| Extract (Analyze) | `DiffbotExtractTool`, `DiffbotExtractLoader` |
|
|
10
|
+
| NLP entities | `DiffbotEntitiesTool` |
|
|
11
|
+
| Crawl | `DiffbotCrawlLoader` |
|
|
12
|
+
| LLM RAG (`ask`) | `ChatDiffbot` (with native streaming) |
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install langchain-diffbot
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Authentication
|
|
21
|
+
|
|
22
|
+
Get an API token at https://app.diffbot.com/get-started/ and export it:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
export DIFFBOT_API_TOKEN="..."
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Every class also accepts `diffbot_api_token=...` directly, or a pre-built `diffbot.Diffbot` client via `client=...` (see [Bring-your-own-client](#bring-your-own-client) below).
|
|
29
|
+
|
|
30
|
+
## Quickstart — Knowledge Graph retriever
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
34
|
+
|
|
35
|
+
retriever = DiffbotKnowledgeGraphRetriever(k=5)
|
|
36
|
+
docs = retriever.invoke("type:Organization industries:\"Artificial Intelligence\" location.city.name:\"Boston\"")
|
|
37
|
+
for d in docs:
|
|
38
|
+
print(d.metadata["name"], "—", d.page_content[:120])
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The query string is a [DQL (Diffbot Query Language)](https://docs.diffbot.com/reference/dql-quickstart) expression.
|
|
42
|
+
|
|
43
|
+
## Shaping the output
|
|
44
|
+
|
|
45
|
+
Diffbot KG entities and web-search results are large. Dumping them straight into an LLM prompt can blow past per-minute input-token limits in a single call. Both retrievers expose three shaping knobs:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from langchain_core.documents import Document
|
|
49
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
50
|
+
|
|
51
|
+
# 1. Project only the top-level fields you care about. Drops everything else
|
|
52
|
+
# from `metadata`. Recommended for agent / tool-use scenarios.
|
|
53
|
+
retriever = DiffbotKnowledgeGraphRetriever(
|
|
54
|
+
k=5,
|
|
55
|
+
fields=["id", "type", "name", "homepageUri", "nbEmployees"],
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# 2. Choose which field becomes `page_content`. First non-empty value wins.
|
|
59
|
+
retriever = DiffbotKnowledgeGraphRetriever(
|
|
60
|
+
content_fields=["summary", "description", "name"],
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# 3. For total control, pass a `document_mapper` that turns a raw entity
|
|
64
|
+
# dict into whatever Document shape you want.
|
|
65
|
+
def mapper(entity: dict) -> Document:
|
|
66
|
+
return Document(
|
|
67
|
+
page_content=entity.get("summary", ""),
|
|
68
|
+
metadata={"id": entity["id"], "name": entity["name"]},
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
retriever = DiffbotKnowledgeGraphRetriever(document_mapper=mapper)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
`fields` and `content_fields` are ignored when `document_mapper` is set. The same knobs work on `DiffbotWebSearchRetriever`.
|
|
75
|
+
|
|
76
|
+
## Web search
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
from langchain_diffbot import DiffbotWebSearchRetriever
|
|
80
|
+
|
|
81
|
+
web = DiffbotWebSearchRetriever(k=5, fields=["title", "pageUrl", "score"])
|
|
82
|
+
docs = web.invoke("diffbot knowledge graph llm grounding")
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Extract a URL
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from langchain_diffbot import DiffbotExtractTool, DiffbotExtractLoader
|
|
89
|
+
|
|
90
|
+
# Single URL
|
|
91
|
+
tool = DiffbotExtractTool()
|
|
92
|
+
page = tool.invoke({"url": "https://www.diffbot.com/products/extract/"})
|
|
93
|
+
|
|
94
|
+
# Batch — yields one Document per URL, sync or async
|
|
95
|
+
loader = DiffbotExtractLoader(urls=["https://example.com", "https://diffbot.com"])
|
|
96
|
+
for doc in loader.lazy_load():
|
|
97
|
+
print(doc.metadata["title"], doc.page_content[:200])
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
`DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`.
|
|
101
|
+
|
|
102
|
+
## ChatDiffbot
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from langchain_core.messages import HumanMessage
|
|
106
|
+
from langchain_diffbot import ChatDiffbot
|
|
107
|
+
|
|
108
|
+
llm = ChatDiffbot()
|
|
109
|
+
|
|
110
|
+
for chunk in llm.stream([HumanMessage(content="What is the Diffbot Knowledge Graph?")]):
|
|
111
|
+
print(chunk.content, end="", flush=True)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
`_stream` / `_astream` are native — no thread-pool fallback. `.invoke()` aggregates the stream into a single message.
|
|
115
|
+
|
|
116
|
+
## Using a retriever in a chain
|
|
117
|
+
|
|
118
|
+
The retrievers are standard `BaseRetriever`s, so they slot into LCEL like any other:
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from langchain_anthropic import ChatAnthropic
|
|
122
|
+
from langchain_core.output_parsers import StrOutputParser
|
|
123
|
+
from langchain_core.prompts import ChatPromptTemplate
|
|
124
|
+
from langchain_core.runnables import RunnablePassthrough
|
|
125
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
126
|
+
|
|
127
|
+
retriever = DiffbotKnowledgeGraphRetriever(
|
|
128
|
+
k=5,
|
|
129
|
+
fields=["id", "name", "homepageUri", "nbEmployees", "industries"],
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
prompt = ChatPromptTemplate.from_template(
|
|
133
|
+
"Answer using only this Diffbot KG context:\n\n{context}\n\nQuestion: {question}"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _format(docs):
|
|
138
|
+
return "\n---\n".join(
|
|
139
|
+
f"{d.metadata.get('name')} (id={d.metadata.get('id')}): {d.page_content}"
|
|
140
|
+
for d in docs
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
chain = (
|
|
145
|
+
{"context": retriever | _format, "question": RunnablePassthrough()}
|
|
146
|
+
| prompt
|
|
147
|
+
| ChatAnthropic(model="claude-sonnet-4-6")
|
|
148
|
+
| StrOutputParser()
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
chain.invoke('type:Organization location.city.name:"Boston" industries:"Biotech"')
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## Bring-your-own-client
|
|
155
|
+
|
|
156
|
+
Every class accepts a pre-built `diffbot.Diffbot` (or `diffbot.DiffbotAsync`) via `client` / `async_client`. The package uses it as-is and **does not close it** — you own the lifecycle. This is the escape hatch for anything the SDK supports that's not re-exposed as a field (custom URLs, `transport=`, shared connection pools, custom headers).
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from diffbot import Diffbot
|
|
160
|
+
from langchain_diffbot import DiffbotKnowledgeGraphRetriever
|
|
161
|
+
|
|
162
|
+
# One client shared across many retriever calls (no per-call httpx pool churn)
|
|
163
|
+
shared = Diffbot(token="...", timeout=60.0)
|
|
164
|
+
retriever = DiffbotKnowledgeGraphRetriever(client=shared, k=5)
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
## Examples
|
|
168
|
+
|
|
169
|
+
The [`examples/`](./examples) folder has runnable demos:
|
|
170
|
+
|
|
171
|
+
- [`examples/quickstart/`](./examples/quickstart) — full tour: every public class, output shaping, async, and a multi-tool research agent.
|
|
172
|
+
- [`examples/company_research/`](./examples/company_research) — the same multi-tool agent as a one-shot CLI: `cd examples && python -m company_research "your question"`. The agent combines KG search + web search + URL extract.
|
|
173
|
+
|
|
174
|
+
Both need `langchain` + `langchain-anthropic` on top of the base package — install the extra:
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
pip install "langchain-diffbot[examples]"
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## Development
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
uv sync --all-groups
|
|
184
|
+
uv run pytest tests/unit_tests
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## Releasing
|
|
188
|
+
|
|
189
|
+
Tokens are stored in macOS Keychain so the Makefile can pull them automatically — no plaintext on disk, no shell-history leaks. First-time setup:
|
|
190
|
+
|
|
191
|
+
```bash
|
|
192
|
+
make set-token-testpypi # prompts; input is hidden as you paste
|
|
193
|
+
make set-token-pypi # same, for real PyPI
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Both targets read with `bash read -rsp` (hidden input), overwrite any existing entry, and never put the token in `make` output or shell history. Re-run either one any time you rotate a token.
|
|
197
|
+
|
|
198
|
+
Then the release flow per version:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
# 1. Bump the version (edits pyproject.toml in place via `uv version --bump`)
|
|
202
|
+
make bump-patch # 0.1.0 → 0.1.1
|
|
203
|
+
# or: make bump-minor # 0.1.0 → 0.2.0
|
|
204
|
+
# or: make bump-major # 0.1.0 → 1.0.0
|
|
205
|
+
|
|
206
|
+
# 2. Publish to TestPyPI and verify installable
|
|
207
|
+
make release-test
|
|
208
|
+
make verify-release-test
|
|
209
|
+
|
|
210
|
+
# 3. Publish to real PyPI (prompts for the version to confirm)
|
|
211
|
+
make release
|
|
212
|
+
make verify-release
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
`make release-test` and `make release` will refuse to publish if the current `pyproject.toml` version is already on the target index — so the workflow is "bump → release-test → verify → release → verify".
|
|
216
|
+
|
|
217
|
+
To rotate: revoke the old token at https://pypi.org/manage/account/token/ (or the TestPyPI equivalent), then run `make set-token-pypi` / `make set-token-testpypi` again — it overwrites the existing Keychain entry without prompting.
|
|
218
|
+
|
|
219
|
+
Integration tests hit the live Diffbot API and require `DIFFBOT_API_TOKEN`:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
uv run pytest tests/integration_tests
|
|
223
|
+
```
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""LangChain integration for Diffbot.
|
|
2
|
+
|
|
3
|
+
Thin layer over the official `diffbot-python` SDK. Every public class accepts
|
|
4
|
+
either a `diffbot_api_token` (or `DIFFBOT_API_TOKEN` env var) or a pre-built
|
|
5
|
+
`diffbot.Diffbot` / `diffbot.DiffbotAsync` client via the `client` /
|
|
6
|
+
`async_client` fields — anything the SDK can do, you can do via these classes.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from langchain_diffbot.chat_models import ChatDiffbot
|
|
10
|
+
from langchain_diffbot.document_loaders import (
|
|
11
|
+
DiffbotCrawlLoader,
|
|
12
|
+
DiffbotExtractLoader,
|
|
13
|
+
)
|
|
14
|
+
from langchain_diffbot.retrievers import (
|
|
15
|
+
DiffbotKnowledgeGraphRetriever,
|
|
16
|
+
DiffbotWebSearchRetriever,
|
|
17
|
+
)
|
|
18
|
+
from langchain_diffbot.tools import (
|
|
19
|
+
DiffbotDQLProbeTool,
|
|
20
|
+
DiffbotEntitiesTool,
|
|
21
|
+
DiffbotExtractTool,
|
|
22
|
+
DiffbotKnowledgeGraphTool,
|
|
23
|
+
DiffbotOntologyTool,
|
|
24
|
+
DiffbotWebSearchTool,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"ChatDiffbot",
|
|
29
|
+
"DiffbotCrawlLoader",
|
|
30
|
+
"DiffbotDQLProbeTool",
|
|
31
|
+
"DiffbotEntitiesTool",
|
|
32
|
+
"DiffbotExtractLoader",
|
|
33
|
+
"DiffbotExtractTool",
|
|
34
|
+
"DiffbotKnowledgeGraphRetriever",
|
|
35
|
+
"DiffbotKnowledgeGraphTool",
|
|
36
|
+
"DiffbotOntologyTool",
|
|
37
|
+
"DiffbotWebSearchRetriever",
|
|
38
|
+
"DiffbotWebSearchTool",
|
|
39
|
+
]
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Shared base for langchain-diffbot components.
|
|
2
|
+
|
|
3
|
+
Every public class in this package inherits from `_BaseDiffbotComponent`. The
|
|
4
|
+
mixin holds the token / timeout / optional pre-built SDK clients and exposes
|
|
5
|
+
two context managers (`_sync_db`, `_async_db`) that the components use to
|
|
6
|
+
acquire a `diffbot.Diffbot` / `diffbot.DiffbotAsync` for a single call.
|
|
7
|
+
|
|
8
|
+
Bring-your-own-client: if the user supplies `client=...` or
|
|
9
|
+
`async_client=...`, we use it as-is and **do not close it** — the user owns
|
|
10
|
+
the lifecycle. Otherwise we construct a fresh SDK client per call and close
|
|
11
|
+
it on exit (same per-call lifecycle as the previous hand-rolled httpx
|
|
12
|
+
wrapper).
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
from collections.abc import AsyncIterator, Iterator
|
|
19
|
+
from contextlib import asynccontextmanager, contextmanager
|
|
20
|
+
|
|
21
|
+
from diffbot import Diffbot, DiffbotAsync
|
|
22
|
+
from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class _BaseDiffbotComponent(BaseModel):
|
|
26
|
+
"""Mixin holding token, timeout, and optional pre-built SDK clients.
|
|
27
|
+
|
|
28
|
+
Concrete classes inherit from this *and* a LangChain base
|
|
29
|
+
(`BaseRetriever`, `BaseTool`, `BaseDocumentLoader`, `BaseChatModel`).
|
|
30
|
+
Both are Pydantic models, so their fields merge.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
34
|
+
|
|
35
|
+
diffbot_api_token: SecretStr | None = Field(default=None)
|
|
36
|
+
"""Diffbot API token. Falls back to `DIFFBOT_API_TOKEN`.
|
|
37
|
+
|
|
38
|
+
Not required when both `client` and `async_client` are supplied.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
timeout: float = 30.0
|
|
42
|
+
"""HTTP timeout (seconds) for SDK clients we construct ourselves.
|
|
43
|
+
|
|
44
|
+
Ignored when `client` / `async_client` are supplied.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
client: Diffbot | None = Field(default=None, exclude=True, repr=False)
|
|
48
|
+
"""Optional pre-built sync SDK client.
|
|
49
|
+
|
|
50
|
+
If set, we use it as-is and do not close it.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
async_client: DiffbotAsync | None = Field(default=None, exclude=True, repr=False)
|
|
54
|
+
"""Optional pre-built async SDK client.
|
|
55
|
+
|
|
56
|
+
If set, we use it as-is and do not close it.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
@model_validator(mode="after")
|
|
60
|
+
def _resolve_token(self) -> _BaseDiffbotComponent:
|
|
61
|
+
# If the user gave us a client (for either side), we can't be sure
|
|
62
|
+
# they'll use the other side — but token resolution shouldn't block
|
|
63
|
+
# construction in that case. Defer the missing-token error to call time.
|
|
64
|
+
if self.client is not None or self.async_client is not None:
|
|
65
|
+
return self
|
|
66
|
+
if (
|
|
67
|
+
self.diffbot_api_token is None
|
|
68
|
+
or not self.diffbot_api_token.get_secret_value()
|
|
69
|
+
):
|
|
70
|
+
env_token = os.environ.get("DIFFBOT_API_TOKEN", "")
|
|
71
|
+
if not env_token:
|
|
72
|
+
msg = (
|
|
73
|
+
"A Diffbot API token is required. Pass `diffbot_api_token=...`, "
|
|
74
|
+
"set the `DIFFBOT_API_TOKEN` environment variable, or supply a "
|
|
75
|
+
"pre-built `client` / `async_client`."
|
|
76
|
+
)
|
|
77
|
+
raise ValueError(msg)
|
|
78
|
+
self.diffbot_api_token = SecretStr(env_token)
|
|
79
|
+
return self
|
|
80
|
+
|
|
81
|
+
def _token(self) -> str:
|
|
82
|
+
if (
|
|
83
|
+
self.diffbot_api_token is None
|
|
84
|
+
or not self.diffbot_api_token.get_secret_value()
|
|
85
|
+
):
|
|
86
|
+
msg = (
|
|
87
|
+
"A Diffbot API token is required for this call. Pass "
|
|
88
|
+
"`diffbot_api_token=...`, set `DIFFBOT_API_TOKEN`, or supply a "
|
|
89
|
+
"pre-built client."
|
|
90
|
+
)
|
|
91
|
+
raise ValueError(msg)
|
|
92
|
+
return self.diffbot_api_token.get_secret_value()
|
|
93
|
+
|
|
94
|
+
@contextmanager
|
|
95
|
+
def _sync_db(self) -> Iterator[Diffbot]:
|
|
96
|
+
"""Yield a `Diffbot` for one call. Closes only clients we constructed."""
|
|
97
|
+
if self.client is not None:
|
|
98
|
+
yield self.client
|
|
99
|
+
return
|
|
100
|
+
with Diffbot(token=self._token(), timeout=self.timeout) as db:
|
|
101
|
+
yield db
|
|
102
|
+
|
|
103
|
+
@asynccontextmanager
|
|
104
|
+
async def _async_db(self) -> AsyncIterator[DiffbotAsync]:
|
|
105
|
+
"""Yield a `DiffbotAsync` for one call. Closes only clients we constructed."""
|
|
106
|
+
if self.async_client is not None:
|
|
107
|
+
yield self.async_client
|
|
108
|
+
return
|
|
109
|
+
async with DiffbotAsync(token=self._token(), timeout=self.timeout) as db:
|
|
110
|
+
yield db
|