langchain-diffbot 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/PKG-INFO +6 -2
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/README.md +4 -0
- langchain_diffbot-0.3.0/langchain_diffbot/document_loaders.py +313 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/langchain_diffbot/tools.py +12 -2
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/pyproject.toml +11 -3
- langchain_diffbot-0.2.0/langchain_diffbot/document_loaders.py +0 -146
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/.gitignore +0 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/LICENSE +0 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/langchain_diffbot/__init__.py +0 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/langchain_diffbot/_base.py +0 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/langchain_diffbot/chat_models.py +0 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/langchain_diffbot/py.typed +0 -0
- {langchain_diffbot-0.2.0 → langchain_diffbot-0.3.0}/langchain_diffbot/retrievers.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: langchain-diffbot
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: LangChain integration for the Diffbot Knowledge Graph and Extract APIs
|
|
5
5
|
Project-URL: Homepage, https://www.diffbot.com/
|
|
6
6
|
Project-URL: Repository, https://github.com/diffbot/langchain-diffbot
|
|
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
19
19
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
20
|
Requires-Python: <4.0,>=3.10
|
|
21
|
-
Requires-Dist: diffbot
|
|
21
|
+
Requires-Dist: diffbot<4,>=3.4.0
|
|
22
22
|
Requires-Dist: httpx<1.0,>=0.27
|
|
23
23
|
Requires-Dist: langchain-core<2.0,>=1.0
|
|
24
24
|
Provides-Extra: examples
|
|
@@ -156,6 +156,8 @@ for doc in loader.lazy_load():
|
|
|
156
156
|
|
|
157
157
|
`DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`.
|
|
158
158
|
|
|
159
|
+
Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`.
|
|
160
|
+
|
|
159
161
|
## Crawl a site
|
|
160
162
|
|
|
161
163
|
`DiffbotCrawlLoader` drives a Diffbot crawl job and yields one `Document` per crawled URL. The `page_content` is the URL itself (the crawl API surfaces URLs, not page contents) — chain it with `DiffbotExtractLoader` to fetch the content of each URL.
|
|
@@ -171,6 +173,8 @@ for doc in loader.lazy_load():
|
|
|
171
173
|
print(doc.metadata["url"], doc.metadata["status"])
|
|
172
174
|
```
|
|
173
175
|
|
|
176
|
+
By default the loader watches the job. Pass `download=True` for an opt-in start / poll / download path: it starts a job (or reuses `job_name`), polls until the job finishes (`poll_interval`, `poll_timeout`), then calls `crawl_download` and yields one `Document` per extracted record with real page content. `download_type`, `download_num`, and `record_mapper` tune the download.
|
|
177
|
+
|
|
174
178
|
## ChatDiffbot
|
|
175
179
|
|
|
176
180
|
```python
|
|
@@ -124,6 +124,8 @@ for doc in loader.lazy_load():
|
|
|
124
124
|
|
|
125
125
|
`DiffbotExtractTool` returns a structured `{"error": ..., "errorCode": ...}` dict when Diffbot reports an extraction failure (200 with `errorCode`), so agents can react and try another URL instead of catching an exception. Auth / rate-limit errors propagate as `diffbot.errors.AuthError` / `RateLimitError`.
|
|
126
126
|
|
|
127
|
+
Both `DiffbotExtractTool` and `DiffbotExtractLoader` take an `extract_options` dict that is forwarded to `Diffbot.extract` on every call: `timeout`, `mode`, `fields`, `fallback`, `discussion`, `paging`, `max_pages`, `natural_language`, `summary_num_sentences`, `max_tags`, `tag_confidence`, `category_confidence`, `use_proxy`, and `extra` (for `proxy` / `proxyAuth`). It is empty by default. Requires `diffbot>=3.4.0,<4`.
|
|
128
|
+
|
|
127
129
|
## Crawl a site
|
|
128
130
|
|
|
129
131
|
`DiffbotCrawlLoader` drives a Diffbot crawl job and yields one `Document` per crawled URL. The `page_content` is the URL itself (the crawl API surfaces URLs, not page contents) — chain it with `DiffbotExtractLoader` to fetch the content of each URL.
|
|
@@ -139,6 +141,8 @@ for doc in loader.lazy_load():
|
|
|
139
141
|
print(doc.metadata["url"], doc.metadata["status"])
|
|
140
142
|
```
|
|
141
143
|
|
|
144
|
+
By default the loader watches the job. Pass `download=True` for an opt-in start / poll / download path: it starts a job (or reuses `job_name`), polls until the job finishes (`poll_interval`, `poll_timeout`), then calls `crawl_download` and yields one `Document` per extracted record with real page content. `download_type`, `download_num`, and `record_mapper` tune the download.
|
|
145
|
+
|
|
142
146
|
## ChatDiffbot
|
|
143
147
|
|
|
144
148
|
```python
|
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
"""Diffbot document loaders — batch ingestion via Extract and Crawl."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import time
|
|
7
|
+
from collections.abc import AsyncIterator, Callable, Iterator
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from diffbot.crawl import CrawlEvent, CrawlEventType
|
|
11
|
+
from diffbot.errors import JobError
|
|
12
|
+
from langchain_core.document_loaders import BaseLoader
|
|
13
|
+
from langchain_core.documents import Document
|
|
14
|
+
from pydantic import Field, field_validator
|
|
15
|
+
|
|
16
|
+
from langchain_diffbot._base import _BaseDiffbotComponent
|
|
17
|
+
|
|
18
|
+
ExtractDocumentMapper = Callable[[str, dict[str, Any]], Document]
|
|
19
|
+
"""`(url, raw_response) -> Document`."""
|
|
20
|
+
|
|
21
|
+
CrawlEventMapper = Callable[[CrawlEvent], Document | None]
|
|
22
|
+
"""`(event) -> Document | None`. Return None to skip the event."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _default_extract_mapper(url: str, raw: dict[str, Any]) -> Document:
|
|
26
|
+
objects = raw.get("objects") or []
|
|
27
|
+
first = objects[0] if objects else {}
|
|
28
|
+
page_content = first.get("text") or raw.get("markdown") or ""
|
|
29
|
+
metadata = {
|
|
30
|
+
"url": url,
|
|
31
|
+
"title": first.get("title") or raw.get("title"),
|
|
32
|
+
"pageUrl": first.get("pageUrl") or raw.get("url"),
|
|
33
|
+
"resolvedPageUrl": first.get("resolvedPageUrl"),
|
|
34
|
+
"type": first.get("type") or raw.get("type"),
|
|
35
|
+
}
|
|
36
|
+
return Document(page_content=page_content, metadata=metadata)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
CrawlRecordMapper = Callable[[dict[str, Any]], Document | None]
|
|
40
|
+
"""`(record) -> Document | None` for `crawl_download` records. None skips."""
|
|
41
|
+
|
|
42
|
+
# Crawler job status codes that mean "still running" (matches the SDK's own
|
|
43
|
+
# watch loop); anything else is terminal.
|
|
44
|
+
_CRAWL_RUNNING_STATUSES = (0, 7)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _default_record_mapper(record: dict[str, Any]) -> Document | None:
|
|
48
|
+
url = record.get("pageUrl") or record.get("resolvedPageUrl") or ""
|
|
49
|
+
return _default_extract_mapper(url, {"objects": [record]})
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _check_job_done(job_name: str, job: dict[str, Any]) -> bool:
|
|
53
|
+
"""Return True when the job is terminal; raise `JobError` if it failed.
|
|
54
|
+
|
|
55
|
+
`crawl_get_job` returns `{}` for an unknown job; that is an error, not a
|
|
56
|
+
job that is "still running" (otherwise we'd poll until `poll_timeout`).
|
|
57
|
+
"""
|
|
58
|
+
if not job:
|
|
59
|
+
raise JobError(job_name, f"Crawler job {job_name!r} not found")
|
|
60
|
+
status = job.get("jobStatus") or {}
|
|
61
|
+
if status.get("status", 0) in _CRAWL_RUNNING_STATUSES:
|
|
62
|
+
return False
|
|
63
|
+
message = status.get("message", "")
|
|
64
|
+
if "fail" in message.lower() or "error" in message.lower():
|
|
65
|
+
raise JobError(job_name, f"Crawler job failed: {message}")
|
|
66
|
+
return True
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _poll_timeout_error(job_name: str, timeout: float) -> TimeoutError:
|
|
70
|
+
return TimeoutError(f"Crawl job {job_name!r} not finished after {timeout}s")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _default_crawl_mapper(event: CrawlEvent) -> Document | None:
|
|
74
|
+
# Non-URL events (e.g. JOB_CREATED) are skipped by default. The Document's
|
|
75
|
+
# page_content is the URL itself — the crawl SDK only surfaces URLs, not
|
|
76
|
+
# page contents. Chain with `DiffbotExtractLoader` to fetch content.
|
|
77
|
+
if event.event_type != CrawlEventType.URL_PROCESSED:
|
|
78
|
+
return None
|
|
79
|
+
url = event.details.get("url", "")
|
|
80
|
+
return Document(
|
|
81
|
+
page_content=url,
|
|
82
|
+
metadata={
|
|
83
|
+
"url": url,
|
|
84
|
+
"status": event.details.get("status"),
|
|
85
|
+
"crawl_timestamp": event.timestamp,
|
|
86
|
+
},
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class DiffbotExtractLoader(_BaseDiffbotComponent, BaseLoader):
|
|
91
|
+
"""Loader that calls `extract` on each of `urls` and yields a `Document`.
|
|
92
|
+
|
|
93
|
+
Example:
|
|
94
|
+
```python
|
|
95
|
+
from diffbot import Diffbot
|
|
96
|
+
from langchain_diffbot import DiffbotExtractLoader
|
|
97
|
+
|
|
98
|
+
docs = DiffbotExtractLoader(
|
|
99
|
+
client=Diffbot(token=...),
|
|
100
|
+
urls=["https://example.com", "https://diffbot.com"],
|
|
101
|
+
).load()
|
|
102
|
+
```
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
urls: list[str]
|
|
106
|
+
"""URLs to extract."""
|
|
107
|
+
|
|
108
|
+
api: str = "analyze"
|
|
109
|
+
"""Diffbot extract API. Defaults to `analyze`."""
|
|
110
|
+
|
|
111
|
+
fmt: str = "markdown"
|
|
112
|
+
"""Output format. `markdown` uses Diffbot's LLM-optimized mode."""
|
|
113
|
+
|
|
114
|
+
extract_options: dict[str, Any] = Field(default_factory=dict)
|
|
115
|
+
"""Keyword options forwarded to `Diffbot.extract` for every URL.
|
|
116
|
+
|
|
117
|
+
Any of the SDK's extract options (`timeout`, `mode`, `fields`, `fallback`,
|
|
118
|
+
`discussion`, `paging`, `max_pages`, `natural_language`,
|
|
119
|
+
`summary_num_sentences`, `max_tags`, `tag_confidence`,
|
|
120
|
+
`category_confidence`, `use_proxy`, `extra`). Empty by default.
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
document_mapper: ExtractDocumentMapper | None = None
|
|
124
|
+
"""Optional `(url, raw_response) -> Document` override."""
|
|
125
|
+
|
|
126
|
+
def _to_doc(self, url: str, raw: dict[str, Any]) -> Document:
|
|
127
|
+
mapper = self.document_mapper or _default_extract_mapper
|
|
128
|
+
return mapper(url, raw)
|
|
129
|
+
|
|
130
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
131
|
+
"""Yield one `Document` per URL, calling `extract` synchronously."""
|
|
132
|
+
with self._sync_db() as db:
|
|
133
|
+
for url in self.urls:
|
|
134
|
+
raw = db.extract(
|
|
135
|
+
url, api=self.api, fmt=self.fmt, **self.extract_options
|
|
136
|
+
)
|
|
137
|
+
yield self._to_doc(url, raw)
|
|
138
|
+
|
|
139
|
+
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
140
|
+
"""Yield one `Document` per URL, calling `extract` asynchronously."""
|
|
141
|
+
async with self._async_db() as db:
|
|
142
|
+
for url in self.urls:
|
|
143
|
+
raw = await db.extract(
|
|
144
|
+
url, api=self.api, fmt=self.fmt, **self.extract_options
|
|
145
|
+
)
|
|
146
|
+
yield self._to_doc(url, raw)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader):
|
|
150
|
+
"""Loader that drives a Diffbot crawl and yields a `Document` per URL.
|
|
151
|
+
|
|
152
|
+
By default `page_content` is the crawled URL itself (the crawl SDK only
|
|
153
|
+
yields URL events, not page contents). To fetch content per URL, chain
|
|
154
|
+
this with `DiffbotExtractLoader`.
|
|
155
|
+
|
|
156
|
+
Defaults `watch=True` so URL events are actually emitted — with
|
|
157
|
+
`watch=False` the SDK only yields a single JOB_CREATED event, which the
|
|
158
|
+
default mapper skips.
|
|
159
|
+
"""
|
|
160
|
+
|
|
161
|
+
site: str
|
|
162
|
+
"""Seed URL for the crawl."""
|
|
163
|
+
|
|
164
|
+
crawl_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
165
|
+
"""Extra kwargs passed to `Diffbot.crawl(site, **crawl_kwargs)`.
|
|
166
|
+
|
|
167
|
+
Defaults to `{"watch": True}` if not overridden.
|
|
168
|
+
"""
|
|
169
|
+
|
|
170
|
+
event_mapper: CrawlEventMapper | None = None
|
|
171
|
+
"""Optional `(event) -> Document | None` override. Return None to skip."""
|
|
172
|
+
|
|
173
|
+
download: bool = False
|
|
174
|
+
"""Opt in to the start / poll / download path instead of watching.
|
|
175
|
+
|
|
176
|
+
When True the loader starts a job (or reuses `job_name`), polls
|
|
177
|
+
`crawl_get_job` until it finishes, then calls `crawl_download` and yields a
|
|
178
|
+
`Document` per extracted record with real page content. `watch` and
|
|
179
|
+
`event_mapper` are not used in this mode. Default False keeps the
|
|
180
|
+
URL-per-event watch behavior.
|
|
181
|
+
"""
|
|
182
|
+
|
|
183
|
+
job_name: str | None = None
|
|
184
|
+
"""Existing crawl job to poll and download (only with `download=True`).
|
|
185
|
+
|
|
186
|
+
If unset, a new job is started for `site` (using `crawl_kwargs`, with
|
|
187
|
+
`watch=False`; set `job_name` inside `crawl_kwargs` to choose its name).
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
poll_interval: float = 2.0
|
|
191
|
+
"""Seconds between job-status polls (only with `download=True`)."""
|
|
192
|
+
|
|
193
|
+
poll_timeout: float | None = 600.0
|
|
194
|
+
"""Max seconds to wait for the job to finish; None waits forever."""
|
|
195
|
+
|
|
196
|
+
download_type: str | None = None
|
|
197
|
+
"""Report passed to `crawl_download(type=...)`.
|
|
198
|
+
|
|
199
|
+
`"urls"` is rejected: that report is CSV text, not records. Use the default
|
|
200
|
+
watch mode (`download=False`) for a URL-per-`Document` crawl.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
download_num: int | None = None
|
|
204
|
+
"""Cap on records passed to `crawl_download(num=...)`."""
|
|
205
|
+
|
|
206
|
+
record_mapper: CrawlRecordMapper | None = None
|
|
207
|
+
"""Optional `(record) -> Document | None` override for downloaded records."""
|
|
208
|
+
|
|
209
|
+
@field_validator("download_type")
|
|
210
|
+
@classmethod
|
|
211
|
+
def _reject_urls_report(cls, v: str | None) -> str | None:
|
|
212
|
+
if v == "urls":
|
|
213
|
+
raise ValueError(
|
|
214
|
+
'download_type="urls" returns CSV text, not records; use the '
|
|
215
|
+
"default watch mode (download=False) to get one Document per URL"
|
|
216
|
+
)
|
|
217
|
+
return v
|
|
218
|
+
|
|
219
|
+
def _kwargs(self) -> dict[str, Any]:
|
|
220
|
+
kw = dict(self.crawl_kwargs)
|
|
221
|
+
kw.setdefault("watch", True)
|
|
222
|
+
return kw
|
|
223
|
+
|
|
224
|
+
def _map_event(self, event: CrawlEvent) -> Document | None:
|
|
225
|
+
mapper = self.event_mapper or _default_crawl_mapper
|
|
226
|
+
return mapper(event)
|
|
227
|
+
|
|
228
|
+
def _records_to_docs(self, records: list[dict[str, Any]]) -> Iterator[Document]:
|
|
229
|
+
mapper = self.record_mapper or _default_record_mapper
|
|
230
|
+
for record in records:
|
|
231
|
+
doc = mapper(record)
|
|
232
|
+
if doc is not None:
|
|
233
|
+
yield doc
|
|
234
|
+
|
|
235
|
+
def _deadline(self) -> float | None:
|
|
236
|
+
if self.poll_timeout is None:
|
|
237
|
+
return None
|
|
238
|
+
return time.monotonic() + self.poll_timeout
|
|
239
|
+
|
|
240
|
+
def _start_kwargs(self) -> dict[str, Any]:
|
|
241
|
+
kw = dict(self.crawl_kwargs)
|
|
242
|
+
kw["watch"] = False
|
|
243
|
+
return kw
|
|
244
|
+
|
|
245
|
+
def lazy_load(self) -> Iterator[Document]:
|
|
246
|
+
"""Drive a crawl synchronously and yield a `Document` per mapped event.
|
|
247
|
+
|
|
248
|
+
With `download=True`, start (or reuse) a job, poll until it finishes,
|
|
249
|
+
and yield a `Document` per downloaded record instead.
|
|
250
|
+
"""
|
|
251
|
+
if self.download:
|
|
252
|
+
yield from self._download_sync()
|
|
253
|
+
return
|
|
254
|
+
with self._sync_db() as db:
|
|
255
|
+
for event in db.crawl(self.site, **self._kwargs()):
|
|
256
|
+
doc = self._map_event(event)
|
|
257
|
+
if doc is not None:
|
|
258
|
+
yield doc
|
|
259
|
+
|
|
260
|
+
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
261
|
+
"""Drive a crawl asynchronously and yield a `Document` per mapped event.
|
|
262
|
+
|
|
263
|
+
With `download=True`, start (or reuse) a job, poll until it finishes,
|
|
264
|
+
and yield a `Document` per downloaded record instead.
|
|
265
|
+
"""
|
|
266
|
+
if self.download:
|
|
267
|
+
async for downloaded in self._download_async():
|
|
268
|
+
yield downloaded
|
|
269
|
+
return
|
|
270
|
+
async with self._async_db() as db:
|
|
271
|
+
async for event in db.crawl(self.site, **self._kwargs()):
|
|
272
|
+
doc = self._map_event(event)
|
|
273
|
+
if doc is not None:
|
|
274
|
+
yield doc
|
|
275
|
+
|
|
276
|
+
def _download_sync(self) -> Iterator[Document]:
|
|
277
|
+
with self._sync_db() as db:
|
|
278
|
+
name = self.job_name
|
|
279
|
+
if name is None:
|
|
280
|
+
for event in db.crawl(self.site, **self._start_kwargs()):
|
|
281
|
+
if event.event_type == CrawlEventType.JOB_CREATED:
|
|
282
|
+
name = event.details["job_name"]
|
|
283
|
+
if name is None:
|
|
284
|
+
raise RuntimeError("Diffbot crawl did not report a job name")
|
|
285
|
+
deadline = self._deadline()
|
|
286
|
+
while not _check_job_done(name, db.crawl_get_job(name)):
|
|
287
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
288
|
+
raise _poll_timeout_error(name, self.poll_timeout or 0)
|
|
289
|
+
time.sleep(self.poll_interval)
|
|
290
|
+
records = db.crawl_download(
|
|
291
|
+
name, format="json", type=self.download_type, num=self.download_num
|
|
292
|
+
)
|
|
293
|
+
yield from self._records_to_docs(records)
|
|
294
|
+
|
|
295
|
+
async def _download_async(self) -> AsyncIterator[Document]:
|
|
296
|
+
async with self._async_db() as db:
|
|
297
|
+
name = self.job_name
|
|
298
|
+
if name is None:
|
|
299
|
+
async for event in db.crawl(self.site, **self._start_kwargs()):
|
|
300
|
+
if event.event_type == CrawlEventType.JOB_CREATED:
|
|
301
|
+
name = event.details["job_name"]
|
|
302
|
+
if name is None:
|
|
303
|
+
raise RuntimeError("Diffbot crawl did not report a job name")
|
|
304
|
+
deadline = self._deadline()
|
|
305
|
+
while not _check_job_done(name, await db.crawl_get_job(name)):
|
|
306
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
307
|
+
raise _poll_timeout_error(name, self.poll_timeout or 0)
|
|
308
|
+
await asyncio.sleep(self.poll_interval)
|
|
309
|
+
records = await db.crawl_download(
|
|
310
|
+
name, format="json", type=self.download_type, num=self.download_num
|
|
311
|
+
)
|
|
312
|
+
for rec_doc in self._records_to_docs(records):
|
|
313
|
+
yield rec_doc
|
|
@@ -53,6 +53,16 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool):
|
|
|
53
53
|
)
|
|
54
54
|
args_schema: type[BaseModel] = _DiffbotExtractInput
|
|
55
55
|
|
|
56
|
+
extract_options: dict[str, Any] = Field(default_factory=dict)
|
|
57
|
+
"""Keyword options forwarded to `Diffbot.extract` on every call.
|
|
58
|
+
|
|
59
|
+
Any of the SDK's extract options (`timeout`, `mode`, `fields`, `fallback`,
|
|
60
|
+
`discussion`, `paging`, `max_pages`, `natural_language`,
|
|
61
|
+
`summary_num_sentences`, `max_tags`, `tag_confidence`,
|
|
62
|
+
`category_confidence`, `use_proxy`, `extra`). Empty by default. Set by the
|
|
63
|
+
developer, not the agent.
|
|
64
|
+
"""
|
|
65
|
+
|
|
56
66
|
@staticmethod
|
|
57
67
|
def _shape_response(raw: dict[str, Any]) -> dict[str, Any]:
|
|
58
68
|
objects = raw.get("objects") or []
|
|
@@ -74,7 +84,7 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool):
|
|
|
74
84
|
) -> dict[str, Any]:
|
|
75
85
|
try:
|
|
76
86
|
with self._sync_db() as db:
|
|
77
|
-
raw = db.extract(url, api=api, fmt=fmt)
|
|
87
|
+
raw = db.extract(url, api=api, fmt=fmt, **self.extract_options)
|
|
78
88
|
except ExtractionError as e:
|
|
79
89
|
return {"error": str(e), "errorCode": e.error_code}
|
|
80
90
|
return self._shape_response(raw)
|
|
@@ -88,7 +98,7 @@ class DiffbotExtractTool(_BaseDiffbotComponent, BaseTool):
|
|
|
88
98
|
) -> dict[str, Any]:
|
|
89
99
|
try:
|
|
90
100
|
async with self._async_db() as db:
|
|
91
|
-
raw = await db.extract(url, api=api, fmt=fmt)
|
|
101
|
+
raw = await db.extract(url, api=api, fmt=fmt, **self.extract_options)
|
|
92
102
|
except ExtractionError as e:
|
|
93
103
|
return {"error": str(e), "errorCode": e.error_code}
|
|
94
104
|
return self._shape_response(raw)
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "langchain-diffbot"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "LangChain integration for the Diffbot Knowledge Graph and Extract APIs"
|
|
9
9
|
license = { text = "MIT" }
|
|
10
10
|
readme = "README.md"
|
|
@@ -24,7 +24,10 @@ requires-python = ">=3.10,<4.0"
|
|
|
24
24
|
dependencies = [
|
|
25
25
|
"langchain-core>=1.0,<2.0",
|
|
26
26
|
"httpx>=0.27,<1.0",
|
|
27
|
-
|
|
27
|
+
# Relock after 3.4.0 is on PyPI: `uv lock --upgrade-package diffbot`
|
|
28
|
+
# (needs the sibling ../langchain checkout). `make dev-sdk` is only for a
|
|
29
|
+
# pre-publish local install; do not commit that path.
|
|
30
|
+
"diffbot>=3.4.0,<4",
|
|
28
31
|
]
|
|
29
32
|
|
|
30
33
|
[project.optional-dependencies]
|
|
@@ -55,6 +58,10 @@ test = [
|
|
|
55
58
|
lint = ["ruff>=0.6,<1.0"]
|
|
56
59
|
typing = ["mypy>=1.10,<2.0"]
|
|
57
60
|
|
|
61
|
+
# Dev-only: the unreleased Diffbot SDK is installed from a sibling checkout by
|
|
62
|
+
# `make dev-sdk` (`uv pip install -e ../diffbot-python`, skipped when that
|
|
63
|
+
# directory is absent). It is deliberately NOT a uv source: a path source can't
|
|
64
|
+
# be conditional on the checkout existing and would break every other install.
|
|
58
65
|
[tool.uv.sources]
|
|
59
66
|
langchain-core = { path = "../langchain/libs/core", editable = true }
|
|
60
67
|
langchain-tests = { path = "../langchain/libs/standard-tests", editable = true }
|
|
@@ -69,7 +76,8 @@ include = ["/langchain_diffbot", "/README.md", "/LICENSE", "/pyproject.toml"]
|
|
|
69
76
|
disallow_untyped_defs = true
|
|
70
77
|
|
|
71
78
|
[[tool.mypy.overrides]]
|
|
72
|
-
# The diffbot SDK doesn't ship a `py.typed` marker yet
|
|
79
|
+
# The diffbot SDK doesn't ship a `py.typed` marker yet (checked against 3.4.0).
|
|
80
|
+
# Drop this override once the SDK ships `py.typed`.
|
|
73
81
|
module = "diffbot.*"
|
|
74
82
|
ignore_missing_imports = true
|
|
75
83
|
|
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
"""Diffbot document loaders — batch ingestion via Extract and Crawl."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
from collections.abc import AsyncIterator, Callable, Iterator
|
|
6
|
-
from typing import Any
|
|
7
|
-
|
|
8
|
-
from diffbot.crawl import CrawlEvent, CrawlEventType
|
|
9
|
-
from langchain_core.document_loaders import BaseLoader
|
|
10
|
-
from langchain_core.documents import Document
|
|
11
|
-
from pydantic import Field
|
|
12
|
-
|
|
13
|
-
from langchain_diffbot._base import _BaseDiffbotComponent
|
|
14
|
-
|
|
15
|
-
ExtractDocumentMapper = Callable[[str, dict[str, Any]], Document]
|
|
16
|
-
"""`(url, raw_response) -> Document`."""
|
|
17
|
-
|
|
18
|
-
CrawlEventMapper = Callable[[CrawlEvent], Document | None]
|
|
19
|
-
"""`(event) -> Document | None`. Return None to skip the event."""
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def _default_extract_mapper(url: str, raw: dict[str, Any]) -> Document:
|
|
23
|
-
objects = raw.get("objects") or []
|
|
24
|
-
first = objects[0] if objects else {}
|
|
25
|
-
page_content = first.get("text") or raw.get("markdown") or ""
|
|
26
|
-
metadata = {
|
|
27
|
-
"url": url,
|
|
28
|
-
"title": first.get("title") or raw.get("title"),
|
|
29
|
-
"pageUrl": first.get("pageUrl") or raw.get("url"),
|
|
30
|
-
"resolvedPageUrl": first.get("resolvedPageUrl"),
|
|
31
|
-
"type": first.get("type") or raw.get("type"),
|
|
32
|
-
}
|
|
33
|
-
return Document(page_content=page_content, metadata=metadata)
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def _default_crawl_mapper(event: CrawlEvent) -> Document | None:
|
|
37
|
-
# Non-URL events (e.g. JOB_CREATED) are skipped by default. The Document's
|
|
38
|
-
# page_content is the URL itself — the crawl SDK only surfaces URLs, not
|
|
39
|
-
# page contents. Chain with `DiffbotExtractLoader` to fetch content.
|
|
40
|
-
if event.event_type != CrawlEventType.URL_PROCESSED:
|
|
41
|
-
return None
|
|
42
|
-
url = event.details.get("url", "")
|
|
43
|
-
return Document(
|
|
44
|
-
page_content=url,
|
|
45
|
-
metadata={
|
|
46
|
-
"url": url,
|
|
47
|
-
"status": event.details.get("status"),
|
|
48
|
-
"crawl_timestamp": event.timestamp,
|
|
49
|
-
},
|
|
50
|
-
)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class DiffbotExtractLoader(_BaseDiffbotComponent, BaseLoader):
|
|
54
|
-
"""Loader that calls `extract` on each of `urls` and yields a `Document`.
|
|
55
|
-
|
|
56
|
-
Example:
|
|
57
|
-
```python
|
|
58
|
-
from diffbot import Diffbot
|
|
59
|
-
from langchain_diffbot import DiffbotExtractLoader
|
|
60
|
-
|
|
61
|
-
docs = DiffbotExtractLoader(
|
|
62
|
-
client=Diffbot(token=...),
|
|
63
|
-
urls=["https://example.com", "https://diffbot.com"],
|
|
64
|
-
).load()
|
|
65
|
-
```
|
|
66
|
-
"""
|
|
67
|
-
|
|
68
|
-
urls: list[str]
|
|
69
|
-
"""URLs to extract."""
|
|
70
|
-
|
|
71
|
-
api: str = "analyze"
|
|
72
|
-
"""Diffbot extract API. Defaults to `analyze`."""
|
|
73
|
-
|
|
74
|
-
fmt: str = "markdown"
|
|
75
|
-
"""Output format. `markdown` uses Diffbot's LLM-optimized mode."""
|
|
76
|
-
|
|
77
|
-
document_mapper: ExtractDocumentMapper | None = None
|
|
78
|
-
"""Optional `(url, raw_response) -> Document` override."""
|
|
79
|
-
|
|
80
|
-
def _to_doc(self, url: str, raw: dict[str, Any]) -> Document:
|
|
81
|
-
mapper = self.document_mapper or _default_extract_mapper
|
|
82
|
-
return mapper(url, raw)
|
|
83
|
-
|
|
84
|
-
def lazy_load(self) -> Iterator[Document]:
|
|
85
|
-
"""Yield one `Document` per URL, calling `extract` synchronously."""
|
|
86
|
-
with self._sync_db() as db:
|
|
87
|
-
for url in self.urls:
|
|
88
|
-
raw = db.extract(url, api=self.api, fmt=self.fmt)
|
|
89
|
-
yield self._to_doc(url, raw)
|
|
90
|
-
|
|
91
|
-
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
92
|
-
"""Yield one `Document` per URL, calling `extract` asynchronously."""
|
|
93
|
-
async with self._async_db() as db:
|
|
94
|
-
for url in self.urls:
|
|
95
|
-
raw = await db.extract(url, api=self.api, fmt=self.fmt)
|
|
96
|
-
yield self._to_doc(url, raw)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
class DiffbotCrawlLoader(_BaseDiffbotComponent, BaseLoader):
|
|
100
|
-
"""Loader that drives a Diffbot crawl and yields a `Document` per URL.
|
|
101
|
-
|
|
102
|
-
By default `page_content` is the crawled URL itself (the crawl SDK only
|
|
103
|
-
yields URL events, not page contents). To fetch content per URL, chain
|
|
104
|
-
this with `DiffbotExtractLoader`.
|
|
105
|
-
|
|
106
|
-
Defaults `watch=True` so URL events are actually emitted — with
|
|
107
|
-
`watch=False` the SDK only yields a single JOB_CREATED event, which the
|
|
108
|
-
default mapper skips.
|
|
109
|
-
"""
|
|
110
|
-
|
|
111
|
-
site: str
|
|
112
|
-
"""Seed URL for the crawl."""
|
|
113
|
-
|
|
114
|
-
crawl_kwargs: dict[str, Any] = Field(default_factory=dict)
|
|
115
|
-
"""Extra kwargs passed to `Diffbot.crawl(site, **crawl_kwargs)`.
|
|
116
|
-
|
|
117
|
-
Defaults to `{"watch": True}` if not overridden.
|
|
118
|
-
"""
|
|
119
|
-
|
|
120
|
-
event_mapper: CrawlEventMapper | None = None
|
|
121
|
-
"""Optional `(event) -> Document | None` override. Return None to skip."""
|
|
122
|
-
|
|
123
|
-
def _kwargs(self) -> dict[str, Any]:
|
|
124
|
-
kw = dict(self.crawl_kwargs)
|
|
125
|
-
kw.setdefault("watch", True)
|
|
126
|
-
return kw
|
|
127
|
-
|
|
128
|
-
def _map_event(self, event: CrawlEvent) -> Document | None:
|
|
129
|
-
mapper = self.event_mapper or _default_crawl_mapper
|
|
130
|
-
return mapper(event)
|
|
131
|
-
|
|
132
|
-
def lazy_load(self) -> Iterator[Document]:
|
|
133
|
-
"""Drive a crawl synchronously and yield a `Document` per mapped event."""
|
|
134
|
-
with self._sync_db() as db:
|
|
135
|
-
for event in db.crawl(self.site, **self._kwargs()):
|
|
136
|
-
doc = self._map_event(event)
|
|
137
|
-
if doc is not None:
|
|
138
|
-
yield doc
|
|
139
|
-
|
|
140
|
-
async def alazy_load(self) -> AsyncIterator[Document]:
|
|
141
|
-
"""Drive a crawl asynchronously and yield a `Document` per mapped event."""
|
|
142
|
-
async with self._async_db() as db:
|
|
143
|
-
async for event in db.crawl(self.site, **self._kwargs()):
|
|
144
|
-
doc = self._map_event(event)
|
|
145
|
-
if doc is not None:
|
|
146
|
-
yield doc
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|