diffbot 3.0.0__py3-none-any.whl → 3.1.0__py3-none-any.whl

diffbot/__init__.py

@@ -13,7 +13,7 @@ def _installed_version(dist: str):
 
 
 # The distribution was renamed from diffbot-python to diffbot. The final
-# diffbot-python release ships this same code (see legacy/diffbot-python), so
+# diffbot-python release ships this same code (tag legacy-diffbot-python-v0.3.0), so
 # fall back to its version when that's what is installed.
 _legacy_version = _installed_version("diffbot-python")
 # "0.0.0" when not installed (e.g. running from a source tree).

diffbot/cli/__init__.py

@@ -359,16 +359,17 @@ def _strip_markdown(text: str) -> str:
 @click.option("-n", "--num-results", type=int, default=None, help="Number of results to return")
 @click.option("-m", "--max-tokens", type=int, default=None, help="Limit total response tokens (for agentic use cases)")
 @click.option("-f", "--format", "fmt", type=click.Choice(["list", "json", "text"]), default="list", help="Output format (text is plain/agent-friendly)")
-def web_search(query: str, num_results, max_tokens, fmt: str):
+@click.option("-s", "--snippet-length", type=click.Choice(["short", "medium", "long", "full"], case_sensitive=False), default=None, help="How much of each page's matching content to return (default: long). short keeps a result to ~1-1.5 KB; full returns the whole stored page, for url: lookups")
+def web_search(query: str, num_results, max_tokens, fmt: str, snippet_length):
     """Search the web using the Diffbot LLM web search API."""
     db = get_client()
     try:
         if is_interactive:
             with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), transient=True) as progress:
                 progress.add_task(description="Searching...", total=None)
-                data = db.web_search(query, num_results=num_results, max_tokens=max_tokens)
+                data = db.web_search(query, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
         else:
-            data = db.web_search(query, num_results=num_results, max_tokens=max_tokens)
+            data = db.web_search(query, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
 
         if fmt == "json":
             sys.stdout.write(json.dumps(data, indent=2))

diffbot/client.py

@@ -197,9 +197,22 @@ class Diffbot:
         """Download the ontology and return it as a queryable Ontology (no caching)."""
         return _dql_fetch_ontology(self)
 
-    def web_search(self, text: str, *, num_results: Optional[int] = None, max_tokens: Optional[int] = None) -> Dict[str, Any]:
-        """Search the web via the Diffbot LLM web search API."""
-        return _web_search(self, text, num_results=num_results, max_tokens=max_tokens)
+    def web_search(
+        self,
+        text: str,
+        *,
+        num_results: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        snippet_length: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Search the web via the Diffbot LLM web search API.
+
+        snippet_length: "short", "medium", "long" or "full" — how much of each
+        page's matching content to return. Omit for the API default (long).
+        "full" returns the whole stored page (reranking is disabled for it);
+        use it with a url:<URL> query.
+        """
+        return _web_search(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
 
     def entities(self, text: str, *, lang: str = "auto") -> Dict[str, Any]:
         """Identify and resolve entities and sentiment in text using the Diffbot NLP API.
@@ -349,9 +362,22 @@ class DiffbotAsync:
         """Download the ontology and return it as a queryable Ontology (no caching)."""
         return await _dql_fetch_ontology_async(self)
 
-    async def web_search(self, text: str, *, num_results: Optional[int] = None, max_tokens: Optional[int] = None) -> Dict[str, Any]:
-        """Search the web via the Diffbot LLM web search API."""
-        return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens)
+    async def web_search(
+        self,
+        text: str,
+        *,
+        num_results: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        snippet_length: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Search the web via the Diffbot LLM web search API.
+
+        snippet_length: "short", "medium", "long" or "full" — how much of each
+        page's matching content to return. Omit for the API default (long).
+        "full" returns the whole stored page (reranking is disabled for it);
+        use it with a url:<URL> query.
+        """
+        return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
 
     async def entities(self, text: str, *, lang: str = "auto") -> Dict[str, Any]:
         """Identify and resolve entities and sentiment in text using the Diffbot NLP API.

diffbot/web_search.py

@@ -14,6 +14,7 @@ def web_search(
     *,
     num_results: Optional[int] = None,
     max_tokens: Optional[int] = None,
+    snippet_length: Optional[str] = None,
 ) -> Dict[str, Any]:
     headers = {"Authorization": f"Bearer {client.token}"}
     params: Dict[str, Any] = {"text": text}
@@ -21,6 +22,11 @@ def web_search(
         params["size"] = num_results
     if max_tokens is not None:
         params["maxTokens"] = max_tokens
+    if snippet_length is not None:
+        params["snippetLength"] = snippet_length.upper()
+        if params["snippetLength"] == "FULL":
+            # the reranker always chunks; only the unranked path returns the stored page
+            params["rerank"] = "none"
     response = client._http.get(client.web_search_url, headers=headers, params=params)
     client._raise_for_status(response)
     return response.json()
@@ -32,6 +38,7 @@ async def web_search_async(
     *,
     num_results: Optional[int] = None,
     max_tokens: Optional[int] = None,
+    snippet_length: Optional[str] = None,
 ) -> Dict[str, Any]:
     headers = {"Authorization": f"Bearer {client.token}"}
     params: Dict[str, Any] = {"text": text}
@@ -39,6 +46,11 @@ async def web_search_async(
         params["size"] = num_results
     if max_tokens is not None:
         params["maxTokens"] = max_tokens
+    if snippet_length is not None:
+        params["snippetLength"] = snippet_length.upper()
+        if params["snippetLength"] == "FULL":
+            # the reranker always chunks; only the unranked path returns the stored page
+            params["rerank"] = "none"
     response = await client._http.get(client.web_search_url, headers=headers, params=params)
     client._raise_for_status(response)
     return response.json()

{diffbot-3.0.0.dist-info → diffbot-3.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: diffbot
-Version: 3.0.0
+Version: 3.1.0
 Summary: Python client library for Diffbot APIs
 Project-URL: Homepage, https://github.com/diffbot/diffbot-python
 Project-URL: Documentation, https://github.com/diffbot/diffbot-python#readme
@@ -224,6 +224,13 @@ results = db.web_search("diffbot knowledge graph")
 for r in results["search_results"]:
     print(r["score"], r["title"], r["pageUrl"])
     print(r["content"])
+
+# snippet_length controls how much of each page comes back in "content":
+# "short" (~1-1.5 KB per result), "medium", "long" (default), or "full"
+results = db.web_search("diffbot knowledge graph", num_results=5, snippet_length="short")
+
+# "full" with a url: query returns the whole stored page
+page = db.web_search("url:https://en.wikipedia.org/wiki/Knowledge_graph", snippet_length="full")
 ```
 
 ### Entities (NLP)
@@ -385,6 +392,8 @@ db crawl-list-jobs
 db crawl-delete-job crawl-1234567890
 db web-search "diffbot knowledge graph"
 db web-search "diffbot knowledge graph" -n 5 -f json
+db web-search "diffbot knowledge graph" -n 5 -s short
+db web-search "url:https://en.wikipedia.org/wiki/Knowledge_graph" -s full -f json
 db entities "Apple CEO Tim Cook announced record quarterly earnings."
 db entities "Apple CEO Tim Cook announced record quarterly earnings." -f dql
 ```
@@ -408,3 +417,7 @@ environment variable or `~/.diffbot/credentials`:
 ```bash
 DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```
+
+## Acknowledgements
+
+Thanks to [@attilaolah](https://github.com/attilaolah), who published the original `diffbot` package in 2012 and looked after the name for 14 years.

{diffbot-3.0.0.dist-info → diffbot-3.1.0.dist-info}/RECORD

@@ -1,22 +1,22 @@
-diffbot/__init__.py,sha256=LAZr0P760URw34JXMIcN-2fTMIELq_-IUd1gj8KgVfI,1875
+diffbot/__init__.py,sha256=PmwZ25wHDva_CwchhnW8ds3N35v_1K-1yIfr2c5gDPs,1882
 diffbot/_auth.py,sha256=dzNMWqIjtG3YtRBb_EKpPRiBxvZgogxvoE0ABLvzfOk,1251
 diffbot/ask.py,sha256=taLISU0BUHzi9SwESZmgDST8PUHoLGJnI02aHngljQM,7411
-diffbot/client.py,sha256=k3orUmOv1RG_xHInyEt0bS0CIYyTOBW56ZDEMgvX8nc,13379
+diffbot/client.py,sha256=XtqjDfGK78iZwXVmHx6F-XC5TrGL7NjOLEbZNUR0Ivk,14193
 diffbot/crawl.py,sha256=iYMFmf7HKrbefJrGg14VnlfiBFLOE_Z1pfO4Rn_cDXc,8893
 diffbot/errors.py,sha256=5-AceX5MyNVUhe9pvR_4rnQQmBhvLfwWmrRl7dRZUSg,1576
 diffbot/extract.py,sha256=R9SVxaOi4FjHOQIX5ho_75OwaJ7VdX-mdAoV_UY-lrM,1452
 diffbot/kg.py,sha256=WPvy3B_ywdoM0uEmAgN7-3uOvf4aC4l2mjS2v_tWPN4,3925
 diffbot/nlp.py,sha256=lZJW4MkjhVklIEM2OBfhc8LvresXTy7RsuImPteYsOA,1153
 diffbot/ontology.py,sha256=GWD5m2rz7ECWY6RrO1TeN3kk3DYTgmEHz6gFKASMvAU,6283
-diffbot/web_search.py,sha256=RSaEK0pdAdLgtilc500Mf2MNmJ1DmiYYmIlBQBWvOpE,1356
-diffbot/cli/__init__.py,sha256=uCngMioFdxTSeUrBxPrxepVqqwllAjXVr8W-zThcYaE,17435
+diffbot/web_search.py,sha256=0LzHzngjcC-TWhBiiSODNTmgiLv15R7v7YZ_A0aE8UI,1970
+diffbot/cli/__init__.py,sha256=wqvil8RiFJ21UWb8xYxwZdqGIWrBWNSqTCd79SJ4Gb4,17809
 diffbot/cli/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
 diffbot/cli/_common.py,sha256=_s8WRukzSX3krjJW6HJmT7_3hEjNyv1i_y45tm81b2A,654
 diffbot/cli/dql.py,sha256=iIMaGysMWTScnbAHAdWei2iVUJpMV5lA1pk1E9uQg4s,11439
 diffbot/cli/entities.py,sha256=tsHKexF0b6NnsoUEZJUS2rJidTIE8lt6xrcSenPbwtY,5630
 diffbot/cli/ontology.py,sha256=cLuRQ0KTsOJnFCXdG_zIX9J1ODK8ThPznWVDDEpO1pg,2055
-diffbot-3.0.0.dist-info/METADATA,sha256=-_Ih-DSJGKOuk-FWx-eLELglRvJbb6bj8IN4V12jsBs,13049
-diffbot-3.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-diffbot-3.0.0.dist-info/entry_points.txt,sha256=FCxJhrbl7VNEsTK7zl7qYvXID7gQ1_wxhiw5_Vllb_M,40
-diffbot-3.0.0.dist-info/licenses/LICENSE,sha256=UZlamI1XGeiG0Mit8dsHssNhOuMGKfmNOp5qpf1533w,1063
-diffbot-3.0.0.dist-info/RECORD,,
+diffbot-3.1.0.dist-info/METADATA,sha256=AdpsX46XFeoDRME8-pUjswHSbhW8IWzgGlJDOh2Audo,13747
+diffbot-3.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+diffbot-3.1.0.dist-info/entry_points.txt,sha256=FCxJhrbl7VNEsTK7zl7qYvXID7gQ1_wxhiw5_Vllb_M,40
+diffbot-3.1.0.dist-info/licenses/LICENSE,sha256=UZlamI1XGeiG0Mit8dsHssNhOuMGKfmNOp5qpf1533w,1063
+diffbot-3.1.0.dist-info/RECORD,,

{diffbot-3.0.0.dist-info → diffbot-3.1.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.33.0
 Root-Is-Purelib: true
 Tag: py3-none-any