langchain-diffbot 0.1.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/PKG-INFO +10 -4
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/README.md +7 -1
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/langchain_diffbot/__init__.py +1 -1
- langchain_diffbot-0.3.0/langchain_diffbot/document_loaders.py +313 -0
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/langchain_diffbot/tools.py +12 -2
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/pyproject.toml +11 -3
- langchain_diffbot-0.1.1/langchain_diffbot/document_loaders.py +0 -146
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/.gitignore +0 -0
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/LICENSE +0 -0
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/langchain_diffbot/_base.py +0 -0
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/langchain_diffbot/chat_models.py +0 -0
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/langchain_diffbot/py.typed +0 -0
- {langchain_diffbot-0.1.1 → langchain_diffbot-0.3.0}/langchain_diffbot/retrievers.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: langchain-diffbot
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: LangChain integration for the Diffbot Knowledge Graph and Extract APIs
|
|
5
5
|
Project-URL: Homepage, https://www.diffbot.com/
|
|
6
6
|
Project-URL: Repository, https://github.com/diffbot/langchain-diffbot
|
|
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
19
19
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
20
|
Requires-Python: <4.0,>=3.10
|
|
21
|
-
Requires-Dist: diffbot
|
|
21
|
+
Requires-Dist: diffbot<4,>=3.4.0
|
|
22
22
|
Requires-Dist: httpx<1.0,>=0.27
|
|
23
23
|
Requires-Dist: langchain-core<2.0,>=1.0
|
|
24
24
|
Provides-Extra: examples
|
|
@@ -34,6 +34,8 @@ Description-Content-Type: text/markdown
|
|
|
34
34
|
|
|
35
35
|
[](https://github.com/diffbot/langchain-diffbot/actions/workflows/ci.yml)
|
|
36
36
|
|
|
37
|
+
**LangChain docs:** [https://docs.langchain.com/oss/python/integrations/providers/diffbot](https://docs.langchain.com/oss/python/integrations/providers/diffbot)
|
|
38
|
+
|
|
37
39
|
A thin LangChain integration over the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. Every Diffbot API gets the closest LangChain primitive:
|
|
38
40
|
|
|
39
41
|
| Diffbot API | LangChain class(es) |
|
|
@@ -154,6 +156,8 @@ for doc in loader.lazy_load():
|
|
|
154
156
|
|
|
155
157
|
`DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`.
|
|
156
158
|
|
|
159
|
+
Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`.
|
|
160
|
+
|
|
157
161
|
## Crawl a site
|
|
158
162
|
|
|
159
163
|
`DiffbotCrawlLoader` drives a Diffbot crawl job and yields one `Document` per crawled URL. The `page_content` is the URL itself (the crawl API surfaces URLs, not page contents) — chain it with `DiffbotExtractLoader` to fetch the content of each URL.
|
|
@@ -169,6 +173,8 @@ for doc in loader.lazy_load():
|
|
|
169
173
|
print(doc.metadata["url"], doc.metadata["status"])
|
|
170
174
|
```
|
|
171
175
|
|
|
176
|
+
By default the loader watches the job. Pass `download=True` for an opt-in start / poll / download path: it starts a job (or reuses `job_name`), polls until the job finishes (`poll_interval`, `poll_timeout`), then calls `crawl_download` and yields one `Document` per extracted record with real page content. `download_type`, `download_num`, and `record_mapper` tune the download.
|
|
177
|
+
|
|
172
178
|
## ChatDiffbot
|
|
173
179
|
|
|
174
180
|
```python
|
|
@@ -202,7 +208,7 @@ Every Diffbot API is also exposed as an agent-callable `BaseTool`. Hand a tool-c
|
|
|
202
208
|
|
|
203
209
|
### DiffbotWebSearchTool
|
|
204
210
|
|
|
205
|
-
Runs a Diffbot web search and returns the result list — each item with `title`, `pageUrl`, `score`, and `content`. (Use `DiffbotWebSearchRetriever` when you want `Document` output instead.)
|
|
211
|
+
Runs a [Diffbot web search](https://docs.diffbot.com/reference/web-search-get) and returns the result list — each item with `title`, `pageUrl`, `score`, and `content`. New accounts include 100,000 free web searches per month. (Use `DiffbotWebSearchRetriever` when you want `Document` output instead.)
|
|
206
212
|
|
|
207
213
|
```python
|
|
208
214
|
import os
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://github.com/diffbot/langchain-diffbot/actions/workflows/ci.yml)
|
|
4
4
|
|
|
5
|
+
**LangChain docs:** [https://docs.langchain.com/oss/python/integrations/providers/diffbot](https://docs.langchain.com/oss/python/integrations/providers/diffbot)
|
|
6
|
+
|
|
5
7
|
A thin LangChain integration over the official [`diffbot-python`](https://github.com/diffbot/diffbot-python) SDK. Every Diffbot API gets the closest LangChain primitive:
|
|
6
8
|
|
|
7
9
|
| Diffbot API | LangChain class(es) |
|
|
@@ -122,6 +124,8 @@ for doc in loader.lazy_load():
|
|
|
122
124
|
|
|
123
125
|
`DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`.
|
|
124
126
|
|
|
127
|
+
Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`.
|
|
128
|
+
|
|
125
129
|
## Crawl a site
|
|
126
130
|
|
|
127
131
|
`DiffbotCrawlLoader` drives a Diffbot crawl job and yields one `Document` per crawled URL. The `page_content` is the URL itself (the crawl API surfaces URLs, not page contents) — chain it with `DiffbotExtractLoader` to fetch the content of each URL.
|
|
@@ -137,6 +141,8 @@ for doc in loader.lazy_load():
|
|
|
137
141
|
print(doc.metadata["url"], doc.metadata["status"])
|
|
138
142
|
```
|
|
139
143
|
|
|
144
|
+
By default the loader watches the job. Pass `download=True` for an opt-in start / poll / download path: it starts a job (or reuses `job_name`), polls until the job finishes (`poll_interval`, `poll_timeout`), then calls `crawl_download` and yields one `Document` per extracted record with real page content. `download_type`, `download_num`, and `record_mapper` tune the download.
|
|
145
|
+
|
|
140
146
|
## ChatDiffbot
|
|
141
147
|
|
|
142
148
|
```python
|
|
@@ -170,7 +176,7 @@ Every Diffbot API is also exposed as an agent-callable `BaseTool`. Hand a tool-c
|
|
|
170
176
|
|
|
171
177
|
### DiffbotWebSearchTool
|
|
172
178
|
|
|
173
|
-
Runs a Diffbot web search and returns the result list — each item with `title`, `pageUrl`, `score`, and `content`. (Use `DiffbotWebSearchRetriever` when you want `Document` output instead.)
|
|
179
|
+
Runs a [Diffbot web search](https://docs.diffbot.com/reference/web-search-get) and returns the result list — each item with `title`, `pageUrl`, `score`, and `content`. New accounts include 100,000 free web searches per month. (Use `DiffbotWebSearchRetriever` when you want `Document` output instead.)
|
|
174
180
|
|
|
175
181
|
```python
|
|
176
182
|
import os
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""LangChain integration for Diffbot.
|
|
2
2
|
|
|
3
|
-
Thin layer over the official `diffbot
|
|
3
|
+
Thin layer over the official `diffbot` SDK. Every public class takes a
|
|
4
4
|
pre-built `diffbot.Diffbot` (sync) and/or `diffbot.DiffbotAsync` (async) client
|
|
5
5
|
via the `client` / `async_client` fields — build the client yourself (token,
|
|
6
6
|
`timeout`, `transport=`, custom URLs), pass it in, and share one across many
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"""Diffbot document loaders — batch ingestion via Extract and Crawl."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import AsyncIterator, Callable, Iterator
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from diffbot.crawl import CrawlEvent, CrawlEventType
|
|
11
|
+
from diffbot.errors import JobError
|
|
12
|
+
from langchain_core.document_loaders import BaseLoader
|
|
13
|
+
from langchain_core.documents import Document
|
|
14
|
+
from pydantic import Field, field_validator
|
|
15
|
+
|
|
16
|
+
from langchain_diffbot._base import _BaseDiffbotComponent
|
|
17
|
+
|
|
18
|
+
ExtractDocumentMapper = Callable[[str, dict[str, Any]], Document]
|
|
19
|
+
"""`(url, raw_response) -> Document`."""
|
|
20
|
+
|
|
21
|
+
CrawlEventMapper = Callable[[CrawlEvent], Document | None]
|
|
22
|
+
"""`(event) -> Document | None`. Return None to skip the event."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _default_extract_mapper(url: str, raw: dict[str, Any]) -> Document:
|
|
26
|
+
objects = raw.get("objects") or []
|
|
27
|
+
first = objects[0] if objects else {}
|
|
28
|
+
page_content = first.get("text") or raw.get("markdown") or ""
|
|
29
|
+
metadata = {
|
|
30
|
+
"url": url,
|
|
31
|
+
"title": first.get("title") or raw.get("title"),
|
|
32
|
+
"pageUrl": first.get("pageUrl") or raw.get("url"),
|
|
33
|
+
"resolvedPageUrl": first.get("resolvedPageUrl"),
|
|
34
|
+
"type": first.get("type") or raw.get("type"),
|
|
35
|
+
}
|
|
36
|
+
return Document(page_content=page_content, metadata=metadata)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
CrawlRecordMapper = Callable[[dict[str, Any]], Document | None]
|
|
40
|
+
"""`(record) -> Document | None` for `crawl_download` records. None skips."""
|
|
41
|
+
|
|
42
|
+
# Crawler job status codes that mean "still running" (matches the SDK's own
|
|
43
|
+
# watch loop); anything else is terminal.
|
|
44
|
+
_CRAWL_RUNNING_STATUSES = (0, 7)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _default_record_mapper(record: dict[str, Any]) -> Document | None:
|
|
48
|
+
url = record.get("pageUrl") or record.get("resolvedPageUrl") or ""
|
|
49
|
+
return _default_extract_mapper(url, {"objects": [record]})
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _check_job_done(job_name: str, job: dict[str, Any]) -> bool:
|
|
53
|
+
"""Return True when the job is terminal; raise `JobError` if it failed.
|
|
54
|
+
|
|
55
|
+
`crawl_get_job` returns `{}` for an unknown job; that is an error, not a
|
|
56
|
+
job that is "still running" (otherwise we'd poll until `poll_timeout`).
|
|
57
|
+
"""
|
|
58
|
+
if not job:
|
|
59
|
+
raise JobError(job_name, f"Crawler job {job_name!r} not found")
|
|
60
|
+
status = job.get("jobStatus") or {}
|
|
61
|
+
if status.get("status", 0) in _CRAWL_RUNNING_STATUSES:
|
|
62
|
+
return False
|
|
63
|
+
message = status.get("message", "")
|
|
64
|
+
if "fail" in message.lower() or "error" in message.lower():
|
|
65
|
+
raise JobError(job_name, f"Crawler job failed: {message}")
|
|
66
|
+
return True
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _poll_timeout_error(job_name: str, timeout: float) -> TimeoutError:
|
|
70
|
+
return TimeoutError(f"Crawl job {job_name!r} not finished after {timeout}s")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _default_crawl_mapper(event: CrawlEvent) -> Document | None:
|
|
74
|
+
# Non-URL events (e.g. JOB_CREATED) are skipped by default. The Document's
|
|
75
|
+
# page_content is the URL itself — the crawl SDK only surfaces URLs, not
|
|
76
|
+
# page contents. Chain with `DiffbotExtractLoader` to fetch content.
|
|
77
|
+
if event.event_type != CrawlEventType.URL_PROCESSED:
|
|
78
|
+
return None
|
|
79
|
+
url = event.details.get("url", "")
|
|
80
|
+
return Document(
|
|
81
|
+
page_content=url,
|
|
82
|
+
metadata={
|
|
83
|
+
"url": url,
|
|
84
|
+
"status": event.details.get("status"),
|
|
85
|
+
"crawl_timestamp": event.timestamp,
|
|
86
|
+
},
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class DiffbotExtractLoader(_BaseDiffbotComponent, BaseLoader):
|
|
91
|
+
"""Loader that calls `extract` on each of `urls` and yields a `Document`.
|
|
92
|
+
|
|
93
|
+
Example:
|
|
94
|
+
```python
|
|
95
|
+
from diffbot import Diffbot
|
|
96
|
+
from langchain_diffbot import DiffbotExtractLoader
|
|
97
|
+
|
|
98
|
+
docs = DiffbotExtractLoader(
|
|
99
|
+
client=Diffbot(token=...),
|
|
100
|
+
urls=["https://example.com", "https://diffbot.com"],
|
|
101
|
+
).load()
|
|
102
|
+
```
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
urls: list[str]
|
|
106
|
+
"""URLs to extract."""
|
|
107
|
+
|
|
108
|
+
api: str = "analyze"
|
|
109
|
+
"""Diffbot extract API. Defaults to `analyze`."""
|
|
110
|
+
|
|
111
|
+
fmt: str = "markdown"
|
|
112
|
+
"""Output format. `markdown` uses Diffbot's LLM-optimized mode."""
|
|
113
|
+
|
|
114
|
+
extract_options: dict[str, Any] = Field(default_factory=dict)
|
|
115
|
+
"""Keyword options forwarded to `Diffbot.extract` for every URL.
|
|
116
|
+
|
|
117
|
+
Any of the SDK's extract options (`timeout`, `mode`, `fields`, `fallback`,
|
|
118
|
+
`discussion`, `paging`, `max_pages`, `natural_language`,
|
|
119
|
+
`summary_num_sentences`, `max_tags`, `tag_confidence`,
|
|
120
|
+
`category_confidence`, `use_proxy`, `extra`). Empty by default.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
document_mapper: ExtractDocumentMapper | None = None
|
|
124
|
+
"""Optional `(url, raw_response) -> Document` override."""
|
|
125
|
+
|
|
126
|
+
def _to_doc(self, url: str, raw: dict[str, Any]) -> Document:
|
|
127
|
+
mapper = self.document_mapper or _default_extract_mapper
|
|
128
|
+
return mapper(url, raw)
|
|
129
|
+
|
|
130
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
131
|
+
"""Yield one `Document` per URL, calling `extract` synchronously."""
|
|
132
|
+
with self._sync_db() as db:
|
|
133
|
+
for url in self.urls:
|
|
134
|
+
raw = db.extract(
|
|
135
|
+
url, api=self.api, fmt=self.fmt, **self.extract_options
|
|
136
|
+
)
|
|
137
|
+
yield self._to_doc(url, raw)
|
|
138
|
+
|
|
139
|
+
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
140
|
+
"""Yield one `Document` per URL, calling `extract` asynchronously."""
|
|
141
|
+
async with self._async_db() as db:
|
|
142
|
+
for url in self.urls:
|
|
143
|
+
raw = await db.extract(
|
|
144
|
+
url, api=self.api, fmt=self.fmt, **self.extract_options
|
|
145
|
+
)
|
|
146
|
+
yield self._to_doc(url, raw)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader):
|
|
150
|
+
"""Loader that drives a Diffbot crawl and yields a `Document` per URL.
|
|
151
|
+
|
|
152
|
+
By default `page_content` is the crawled URL itself (the crawl SDK only
|
|
153
|
+
yields URL events, not page contents). To fetch content per URL, chain
|
|
154
|
+
this with `DiffbotExtractLoader`.
|
|
155
|
+
|
|
156
|
+
Defaults `watch=True` so URL events are actually emitted — with
|
|
157
|
+
`watch=False` the SDK only yields a single JOB_CREATED event, which the
|
|
158
|
+
default mapper skips.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
site: str
|
|
162
|
+
"""Seed URL for the crawl."""
|
|
163
|
+
|
|
164
|
+
crawl_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
165
|
+
"""Extra kwargs passed to `Diffbot.crawl(site, **crawl_kwargs)`.
|
|
166
|
+
|
|
167
|
+
Defaults to `{"watch": True}` if not overridden.
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
event_mapper: CrawlEventMapper | None = None
|
|
171
|
+
"""Optional `(event) -> Document | None` override. Return None to skip."""
|
|
172
|
+
|
|
173
|
+
download: bool = False
|
|
174
|
+
"""Opt in to the start / poll / download path instead of watching.
|
|
175
|
+
|
|
176
|
+
When True the loader starts a job (or reuses `job_name`), polls
|
|
177
|
+
`crawl_get_job` until it finishes, then calls `crawl_download` and yields a
|
|
178
|
+
`Document` per extracted record with real page content. `watch` and
|
|
179
|
+
`event_mapper` are not used in this mode. Default False keeps the
|
|
180
|
+
URL-per-event watch behavior.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
job_name: str | None = None
|
|
184
|
+
"""Existing crawl job to poll and download (only with `download=True`).
|
|
185
|
+
|
|
186
|
+
If unset, a new job is started for `site` (using `crawl_kwargs`, with
|
|
187
|
+
`watch=False`; set `job_name` inside `crawl_kwargs` to choose its name).
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
poll_interval: float = 2.0
|
|
191
|
+
"""Seconds between job-status polls (only with `download=True`)."""
|
|
192
|
+
|
|
193
|
+
poll_timeout: float | None = 600.0
|
|
194
|
+
"""Max seconds to wait for the job to finish; None waits forever."""
|
|
195
|
+
|
|
196
|
+
download_type: str | None = None
|
|
197
|
+
"""Report passed to `crawl_download(type=...)`.
|
|
198
|
+
|
|
199
|
+
`"urls"` is rejected: that report is CSV text, not records. Use the default
|
|
200
|
+
watch mode (`download=False`) for a URL-per-`Document` crawl.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
download_num: int | None = None
|
|
204
|
+
"""Cap on records passed to `crawl_download(num=...)`."""
|
|
205
|
+
|
|
206
|
+
record_mapper: CrawlRecordMapper | None = None
|
|
207
|
+
"""Optional `(record) -> Document | None` override for downloaded records."""
|
|
208
|
+
|
|
209
|
+
@field_validator("download_type")
|
|
210
|
+
@classmethod
|
|
211
|
+
def _reject_urls_report(cls, v: str | None) -> str | None:
|
|
212
|
+
if v == "urls":
|
|
213
|
+
raise ValueError(
|
|
214
|
+
'download_type="urls" returns CSV text, not records; use the '
|
|
215
|
+
"default watch mode (download=False) to get one Document per URL"
|
|
216
|
+
)
|
|
217
|
+
return v
|
|
218
|
+
|
|
219
|
+
def _kwargs(self) -> dict[str, Any]:
|
|
220
|
+
kw = dict(self.crawl_kwargs)
|
|
221
|
+
kw.setdefault("watch", True)
|
|
222
|
+
return kw
|
|
223
|
+
|
|
224
|
+
def _map_event(self, event: CrawlEvent) -> Document | None:
|
|
225
|
+
mapper = self.event_mapper or _default_crawl_mapper
|
|
226
|
+
return mapper(event)
|
|
227
|
+
|
|
228
|
+
def _records_to_docs(self, records: list[dict[str, Any]]) -> Iterator[Document]:
|
|
229
|
+
mapper = self.record_mapper or _default_record_mapper
|
|
230
|
+
for record in records:
|
|
231
|
+
doc = mapper(record)
|
|
232
|
+
if doc is not None:
|
|
233
|
+
yield doc
|
|
234
|
+
|
|
235
|
+
def _deadline(self) -> float | None:
|
|
236
|
+
if self.poll_timeout is None:
|
|
237
|
+
return None
|
|
238
|
+
return time.monotonic() + self.poll_timeout
|
|
239
|
+
|
|
240
|
+
def _start_kwargs(self) -> dict[str, Any]:
|
|
241
|
+
kw = dict(self.crawl_kwargs)
|
|
242
|
+
kw["watch"] = False
|
|
243
|
+
return kw
|
|
244
|
+
|
|
245
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
246
|
+
"""Drive a crawl synchronously and yield a `Document` per mapped event.
|
|
247
|
+
|
|
248
|
+
With `download=True`, start (or reuse) a job, poll until it finishes,
|
|
249
|
+
and yield a `Document` per downloaded record instead.
|
|
250
|
+
"""
|
|
251
|
+
if self.download:
|
|
252
|
+
yield from self._download_sync()
|
|
253
|
+
return
|
|
254
|
+
with self._sync_db() as db:
|
|
255
|
+
for event in db.crawl(self.site, **self._kwargs()):
|
|
256
|
+
doc = self._map_event(event)
|
|
257
|
+
if doc is not None:
|
|
258
|
+
yield doc
|
|
259
|
+
|
|
260
|
+
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
261
|
+
"""Drive a crawl asynchronously and yield a `Document` per mapped event.
|
|
262
|
+
|
|
263
|
+
With `download=True`, start (or reuse) a job, poll until it finishes,
|
|
264
|
+
and yield a `Document` per downloaded record instead.
|
|
265
|
+
"""
|
|
266
|
+
if self.download:
|
|
267
|
+
async for downloaded in self._download_async():
|
|
268
|
+
yield downloaded
|
|
269
|
+
return
|
|
270
|
+
async with self._async_db() as db:
|
|
271
|
+
async for event in db.crawl(self.site, **self._kwargs()):
|
|
272
|
+
doc = self._map_event(event)
|
|
273
|
+
if doc is not None:
|
|
274
|
+
yield doc
|
|
275
|
+
|
|
276
|
+
def _download_sync(self) -> Iterator[Document]:
|
|
277
|
+
with self._sync_db() as db:
|
|
278
|
+
name = self.job_name
|
|
279
|
+
if name is None:
|
|
280
|
+
for event in db.crawl(self.site, **self._start_kwargs()):
|
|
281
|
+
if event.event_type == CrawlEventType.JOB_CREATED:
|
|
282
|
+
name = event.details["job_name"]
|
|
283
|
+
if name is None:
|
|
284
|
+
raise RuntimeError("Diffbot crawl did not report a job name")
|
|
285
|
+
deadline = self._deadline()
|
|
286
|
+
while not _check_job_done(name, db.crawl_get_job(name)):
|
|
287
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
288
|
+
raise _poll_timeout_error(name, self.poll_timeout or 0)
|
|
289
|
+
time.sleep(self.poll_interval)
|
|
290
|
+
records = db.crawl_download(
|
|
291
|
+
name, format="json", type=self.download_type, num=self.download_num
|
|
292
|
+
)
|
|
293
|
+
yield from self._records_to_docs(records)
|
|
294
|
+
|
|
295
|
+
async def _download_async(self) -> AsyncIterator[Document]:
|
|
296
|
+
async with self._async_db() as db:
|
|
297
|
+
name = self.job_name
|
|
298
|
+
if name is None:
|
|
299
|
+
async for event in db.crawl(self.site, **self._start_kwargs()):
|
|
300
|
+
if event.event_type == CrawlEventType.JOB_CREATED:
|
|
301
|
+
name = event.details["job_name"]
|
|
302
|
+
if name is None:
|
|
303
|
+
raise RuntimeError("Diffbot crawl did not report a job name")
|
|
304
|
+
deadline = self._deadline()
|
|
305
|
+
while not _check_job_done(name, await db.crawl_get_job(name)):
|
|
306
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
307
|
+
raise _poll_timeout_error(name, self.poll_timeout or 0)
|
|
308
|
+
await asyncio.sleep(self.poll_interval)
|
|
309
|
+
records = await db.crawl_download(
|
|
310
|
+
name, format="json", type=self.download_type, num=self.download_num
|
|
311
|
+
)
|
|
312
|
+
for rec_doc in self._records_to_docs(records):
|
|
313
|
+
yield rec_doc
|
|
@@ -53,6 +53,16 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool):
|
|
|
53
53
|
)
|
|
54
54
|
args_schema: type[BaseModel] = _DiffbotExtractInput
|
|
55
55
|
|
|
56
|
+
extract_options: dict[str, Any] = Field(default_factory=dict)
|
|
57
|
+
"""Keyword options forwarded to `Diffbot.extract` on every call.
|
|
58
|
+
|
|
59
|
+
Any of the SDK's extract options (`timeout`, `mode`, `fields`, `fallback`,
|
|
60
|
+
`discussion`, `paging`, `max_pages`, `natural_language`,
|
|
61
|
+
`summary_num_sentences`, `max_tags`, `tag_confidence`,
|
|
62
|
+
`category_confidence`, `use_proxy`, `extra`). Empty by default. Set by the
|
|
63
|
+
developer, not the agent.
|
|
64
|
+
"""
|
|
65
|
+
|
|
56
66
|
@staticmethod
|
|
57
67
|
def _shape_response(raw: dict[str, Any]) -> dict[str, Any]:
|
|
58
68
|
objects = raw.get("objects") or []
|
|
@@ -74,7 +84,7 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool):
|
|
|
74
84
|
) -> dict[str, Any]:
|
|
75
85
|
try:
|
|
76
86
|
with self._sync_db() as db:
|
|
77
|
-
raw = db.extract(url, api=api, fmt=fmt)
|
|
87
|
+
raw = db.extract(url, api=api, fmt=fmt, **self.extract_options)
|
|
78
88
|
except ExtractionError as e:
|
|
79
89
|
return {"error": str(e), "errorCode": e.error_code}
|
|
80
90
|
return self._shape_response(raw)
|
|
@@ -88,7 +98,7 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool):
|
|
|
88
98
|
) -> dict[str, Any]:
|
|
89
99
|
try:
|
|
90
100
|
async with self._async_db() as db:
|
|
91
|
-
raw = await db.extract(url, api=api, fmt=fmt)
|
|
101
|
+
raw = await db.extract(url, api=api, fmt=fmt, **self.extract_options)
|
|
92
102
|
except ExtractionError as e:
|
|
93
103
|
return {"error": str(e), "errorCode": e.error_code}
|
|
94
104
|
return self._shape_response(raw)
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "langchain-diffbot"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "LangChain integration for the Diffbot Knowledge Graph and Extract APIs"
|
|
9
9
|
license = { text = "MIT" }
|
|
10
10
|
readme = "README.md"
|
|
@@ -24,7 +24,10 @@ requires-python = ">=3.10,<4.0"
|
|
|
24
24
|
dependencies = [
|
|
25
25
|
"langchain-core>=1.0,<2.0",
|
|
26
26
|
"httpx>=0.27,<1.0",
|
|
27
|
-
|
|
27
|
+
# Relock after 3.4.0 is on PyPI: `uv lock --upgrade-package diffbot`
|
|
28
|
+
# (needs the sibling ../langchain checkout). `make dev-sdk` is only for a
|
|
29
|
+
# pre-publish local install; do not commit that path.
|
|
30
|
+
"diffbot>=3.4.0,<4",
|
|
28
31
|
]
|
|
29
32
|
|
|
30
33
|
[project.optional-dependencies]
|
|
@@ -55,6 +58,10 @@ test = [
|
|
|
55
58
|
lint = ["ruff>=0.6,<1.0"]
|
|
56
59
|
typing = ["mypy>=1.10,<2.0"]
|
|
57
60
|
|
|
61
|
+
# Dev-only: the unreleased Diffbot SDK is installed from a sibling checkout by
|
|
62
|
+
# `make dev-sdk` (`uv pip install -e ../diffbot-python`, skipped when that
|
|
63
|
+
# directory is absent). It is deliberately NOT a uv source: a path source can't
|
|
64
|
+
# be conditional on the checkout existing and would break every other install.
|
|
58
65
|
[tool.uv.sources]
|
|
59
66
|
langchain-core = { path = "../langchain/libs/core", editable = true }
|
|
60
67
|
langchain-tests = { path = "../langchain/libs/standard-tests", editable = true }
|
|
@@ -69,7 +76,8 @@ include = ["/langchain_diffbot", "/README.md", "/LICENSE", "/pyproject.toml"]
|
|
|
69
76
|
disallow_untyped_defs = true
|
|
70
77
|
|
|
71
78
|
[[tool.mypy.overrides]]
|
|
72
|
-
# diffbot
|
|
79
|
+
# The diffbot SDK doesn't ship a `py.typed` marker yet (checked against 3.4.0).
|
|
80
|
+
# Drop this override once the SDK ships `py.typed`.
|
|
73
81
|
module = "diffbot.*"
|
|
74
82
|
ignore_missing_imports = true
|
|
75
83
|
|
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
"""Diffbot document loaders — batch ingestion via Extract and Crawl."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from collections.abc import AsyncIterator, Callable, Iterator
|
|
6
|
-
from typing import Any
|
|
7
|
-
|
|
8
|
-
from diffbot.crawl import CrawlEvent, CrawlEventType
|
|
9
|
-
from langchain_core.document_loaders import BaseLoader
|
|
10
|
-
from langchain_core.documents import Document
|
|
11
|
-
from pydantic import Field
|
|
12
|
-
|
|
13
|
-
from langchain_diffbot._base import _BaseDiffbotComponent
|
|
14
|
-
|
|
15
|
-
ExtractDocumentMapper = Callable[[str, dict[str, Any]], Document]
|
|
16
|
-
"""`(url, raw_response) -> Document`."""
|
|
17
|
-
|
|
18
|
-
CrawlEventMapper = Callable[[CrawlEvent], Document | None]
|
|
19
|
-
"""`(event) -> Document | None`. Return None to skip the event."""
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def _default_extract_mapper(url: str, raw: dict[str, Any]) -> Document:
|
|
23
|
-
objects = raw.get("objects") or []
|
|
24
|
-
first = objects[0] if objects else {}
|
|
25
|
-
page_content = first.get("text") or raw.get("markdown") or ""
|
|
26
|
-
metadata = {
|
|
27
|
-
"url": url,
|
|
28
|
-
"title": first.get("title") or raw.get("title"),
|
|
29
|
-
"pageUrl": first.get("pageUrl") or raw.get("url"),
|
|
30
|
-
"resolvedPageUrl": first.get("resolvedPageUrl"),
|
|
31
|
-
"type": first.get("type") or raw.get("type"),
|
|
32
|
-
}
|
|
33
|
-
return Document(page_content=page_content, metadata=metadata)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def _default_crawl_mapper(event: CrawlEvent) -> Document | None:
|
|
37
|
-
# Non-URL events (e.g. JOB_CREATED) are skipped by default. The Document's
|
|
38
|
-
# page_content is the URL itself — the crawl SDK only surfaces URLs, not
|
|
39
|
-
# page contents. Chain with `DiffbotExtractLoader` to fetch content.
|
|
40
|
-
if event.event_type != CrawlEventType.URL_PROCESSED:
|
|
41
|
-
return None
|
|
42
|
-
url = event.details.get("url", "")
|
|
43
|
-
return Document(
|
|
44
|
-
page_content=url,
|
|
45
|
-
metadata={
|
|
46
|
-
"url": url,
|
|
47
|
-
"status": event.details.get("status"),
|
|
48
|
-
"crawl_timestamp": event.timestamp,
|
|
49
|
-
},
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class DiffbotExtractLoader(_BaseDiffbotComponent, BaseLoader):
|
|
54
|
-
"""Loader that calls `extract` on each of `urls` and yields a `Document`.
|
|
55
|
-
|
|
56
|
-
Example:
|
|
57
|
-
```python
|
|
58
|
-
from diffbot import Diffbot
|
|
59
|
-
from langchain_diffbot import DiffbotExtractLoader
|
|
60
|
-
|
|
61
|
-
docs = DiffbotExtractLoader(
|
|
62
|
-
client=Diffbot(token=...),
|
|
63
|
-
urls=["https://example.com", "https://diffbot.com"],
|
|
64
|
-
).load()
|
|
65
|
-
```
|
|
66
|
-
"""
|
|
67
|
-
|
|
68
|
-
urls: list[str]
|
|
69
|
-
"""URLs to extract."""
|
|
70
|
-
|
|
71
|
-
api: str = "analyze"
|
|
72
|
-
"""Diffbot extract API. Defaults to `analyze`."""
|
|
73
|
-
|
|
74
|
-
fmt: str = "markdown"
|
|
75
|
-
"""Output format. `markdown` uses Diffbot's LLM-optimized mode."""
|
|
76
|
-
|
|
77
|
-
document_mapper: ExtractDocumentMapper | None = None
|
|
78
|
-
"""Optional `(url, raw_response) -> Document` override."""
|
|
79
|
-
|
|
80
|
-
def _to_doc(self, url: str, raw: dict[str, Any]) -> Document:
|
|
81
|
-
mapper = self.document_mapper or _default_extract_mapper
|
|
82
|
-
return mapper(url, raw)
|
|
83
|
-
|
|
84
|
-
def lazy_load(self) -> Iterator[Document]:
|
|
85
|
-
"""Yield one `Document` per URL, calling `extract` synchronously."""
|
|
86
|
-
with self._sync_db() as db:
|
|
87
|
-
for url in self.urls:
|
|
88
|
-
raw = db.extract(url, api=self.api, fmt=self.fmt)
|
|
89
|
-
yield self._to_doc(url, raw)
|
|
90
|
-
|
|
91
|
-
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
92
|
-
"""Yield one `Document` per URL, calling `extract` asynchronously."""
|
|
93
|
-
async with self._async_db() as db:
|
|
94
|
-
for url in self.urls:
|
|
95
|
-
raw = await db.extract(url, api=self.api, fmt=self.fmt)
|
|
96
|
-
yield self._to_doc(url, raw)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader):
|
|
100
|
-
"""Loader that drives a Diffbot crawl and yields a `Document` per URL.
|
|
101
|
-
|
|
102
|
-
By default `page_content` is the crawled URL itself (the crawl SDK only
|
|
103
|
-
yields URL events, not page contents). To fetch content per URL, chain
|
|
104
|
-
this with `DiffbotExtractLoader`.
|
|
105
|
-
|
|
106
|
-
Defaults `watch=True` so URL events are actually emitted — with
|
|
107
|
-
`watch=False` the SDK only yields a single JOB_CREATED event, which the
|
|
108
|
-
default mapper skips.
|
|
109
|
-
"""
|
|
110
|
-
|
|
111
|
-
site: str
|
|
112
|
-
"""Seed URL for the crawl."""
|
|
113
|
-
|
|
114
|
-
crawl_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
115
|
-
"""Extra kwargs passed to `Diffbot.crawl(site, **crawl_kwargs)`.
|
|
116
|
-
|
|
117
|
-
Defaults to `{"watch": True}` if not overridden.
|
|
118
|
-
"""
|
|
119
|
-
|
|
120
|
-
event_mapper: CrawlEventMapper | None = None
|
|
121
|
-
"""Optional `(event) -> Document | None` override. Return None to skip."""
|
|
122
|
-
|
|
123
|
-
def _kwargs(self) -> dict[str, Any]:
|
|
124
|
-
kw = dict(self.crawl_kwargs)
|
|
125
|
-
kw.setdefault("watch", True)
|
|
126
|
-
return kw
|
|
127
|
-
|
|
128
|
-
def _map_event(self, event: CrawlEvent) -> Document | None:
|
|
129
|
-
mapper = self.event_mapper or _default_crawl_mapper
|
|
130
|
-
return mapper(event)
|
|
131
|
-
|
|
132
|
-
def lazy_load(self) -> Iterator[Document]:
|
|
133
|
-
"""Drive a crawl synchronously and yield a `Document` per mapped event."""
|
|
134
|
-
with self._sync_db() as db:
|
|
135
|
-
for event in db.crawl(self.site, **self._kwargs()):
|
|
136
|
-
doc = self._map_event(event)
|
|
137
|
-
if doc is not None:
|
|
138
|
-
yield doc
|
|
139
|
-
|
|
140
|
-
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
141
|
-
"""Drive a crawl asynchronously and yield a `Document` per mapped event."""
|
|
142
|
-
async with self._async_db() as db:
|
|
143
|
-
async for event in db.crawl(self.site, **self._kwargs()):
|
|
144
|
-
doc = self._map_event(event)
|
|
145
|
-
if doc is not None:
|
|
146
|
-
yield doc
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|