diffbot 3.0.0__tar.gz → 3.1.0__tar.gz

{diffbot-3.0.0 → diffbot-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: diffbot
-Version: 3.0.0
+Version: 3.1.0
 Summary: Python client library for Diffbot APIs
 Project-URL: Homepage, https://github.com/diffbot/diffbot-python
 Project-URL: Documentation, https://github.com/diffbot/diffbot-python#readme
@@ -224,6 +224,13 @@ results = db.web_search("diffbot knowledge graph")
 for r in results["search_results"]:
     print(r["score"], r["title"], r["pageUrl"])
     print(r["content"])
+
+# snippet_length controls how much of each page comes back in "content":
+# "short" (~1-1.5 KB per result), "medium", "long" (default), or "full"
+results = db.web_search("diffbot knowledge graph", num_results=5, snippet_length="short")
+
+# "full" with a url: query returns the whole stored page
+page = db.web_search("url:https://en.wikipedia.org/wiki/Knowledge_graph", snippet_length="full")
 ```
 
 ### Entities (NLP)
@@ -385,6 +392,8 @@ db crawl-list-jobs
 db crawl-delete-job crawl-1234567890
 db web-search "diffbot knowledge graph"
 db web-search "diffbot knowledge graph" -n 5 -f json
+db web-search "diffbot knowledge graph" -n 5 -s short
+db web-search "url:https://en.wikipedia.org/wiki/Knowledge_graph" -s full -f json
 db entities "Apple CEO Tim Cook announced record quarterly earnings."
 db entities "Apple CEO Tim Cook announced record quarterly earnings." -f dql
 ```
@@ -408,3 +417,7 @@ environment variable or `~/.diffbot/credentials`:
 ```bash
 DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```
+
+## Acknowledgements
+
+Thanks to [@attilaolah](https://github.com/attilaolah), who published the original `diffbot` package in 2012 and looked after the name for 14 years.

{diffbot-3.0.0 → diffbot-3.1.0}/README.md

@@ -188,6 +188,13 @@ results = db.web_search("diffbot knowledge graph")
 for r in results["search_results"]:
     print(r["score"], r["title"], r["pageUrl"])
     print(r["content"])
+
+# snippet_length controls how much of each page comes back in "content":
+# "short" (~1-1.5 KB per result), "medium", "long" (default), or "full"
+results = db.web_search("diffbot knowledge graph", num_results=5, snippet_length="short")
+
+# "full" with a url: query returns the whole stored page
+page = db.web_search("url:https://en.wikipedia.org/wiki/Knowledge_graph", snippet_length="full")
 ```
 
 ### Entities (NLP)
@@ -349,6 +356,8 @@ db crawl-list-jobs
 db crawl-delete-job crawl-1234567890
 db web-search "diffbot knowledge graph"
 db web-search "diffbot knowledge graph" -n 5 -f json
+db web-search "diffbot knowledge graph" -n 5 -s short
+db web-search "url:https://en.wikipedia.org/wiki/Knowledge_graph" -s full -f json
 db entities "Apple CEO Tim Cook announced record quarterly earnings."
 db entities "Apple CEO Tim Cook announced record quarterly earnings." -f dql
 ```
@@ -372,3 +381,7 @@ environment variable or `~/.diffbot/credentials`:
 ```bash
 DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```
+
+## Acknowledgements
+
+Thanks to [@attilaolah](https://github.com/attilaolah), who published the original `diffbot` package in 2012 and looked after the name for 14 years.

{diffbot-3.0.0 → diffbot-3.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "diffbot"
-version = "3.0.0"
+version = "3.1.0"
 description = "Python client library for Diffbot APIs"
 readme = "README.md"
 requires-python = ">=3.9"

{diffbot-3.0.0 → diffbot-3.1.0}/src/diffbot/__init__.py

@@ -13,7 +13,7 @@ def _installed_version(dist: str):
 
 
 # The distribution was renamed from diffbot-python to diffbot. The final
-# diffbot-python release ships this same code (see legacy/diffbot-python), so
+# diffbot-python release ships this same code (tag legacy-diffbot-python-v0.3.0), so
 # fall back to its version when that's what is installed.
 _legacy_version = _installed_version("diffbot-python")
 # "0.0.0" when not installed (e.g. running from a source tree).

{diffbot-3.0.0 → diffbot-3.1.0}/src/diffbot/cli/__init__.py

@@ -359,16 +359,17 @@ def _strip_markdown(text: str) -> str:
 @click.option("-n", "--num-results", type=int, default=None, help="Number of results to return")
 @click.option("-m", "--max-tokens", type=int, default=None, help="Limit total response tokens (for agentic use cases)")
 @click.option("-f", "--format", "fmt", type=click.Choice(["list", "json", "text"]), default="list", help="Output format (text is plain/agent-friendly)")
-def web_search(query: str, num_results, max_tokens, fmt: str):
+@click.option("-s", "--snippet-length", type=click.Choice(["short", "medium", "long", "full"], case_sensitive=False), default=None, help="How much of each page's matching content to return (default: long). short keeps a result to ~1-1.5 KB; full returns the whole stored page, for url: lookups")
+def web_search(query: str, num_results, max_tokens, fmt: str, snippet_length):
     """Search the web using the Diffbot LLM web search API."""
     db = get_client()
     try:
         if is_interactive:
             with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}"), transient=True) as progress:
                 progress.add_task(description="Searching...", total=None)
-                data = db.web_search(query, num_results=num_results, max_tokens=max_tokens)
+                data = db.web_search(query, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
         else:
-            data = db.web_search(query, num_results=num_results, max_tokens=max_tokens)
+            data = db.web_search(query, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
 
         if fmt == "json":
             sys.stdout.write(json.dumps(data, indent=2))

{diffbot-3.0.0 → diffbot-3.1.0}/src/diffbot/client.py

@@ -197,9 +197,22 @@ class Diffbot:
         """Download the ontology and return it as a queryable Ontology (no caching)."""
         return _dql_fetch_ontology(self)
 
-    def web_search(self, text: str, *, num_results: Optional[int] = None, max_tokens: Optional[int] = None) -> Dict[str, Any]:
-        """Search the web via the Diffbot LLM web search API."""
-        return _web_search(self, text, num_results=num_results, max_tokens=max_tokens)
+    def web_search(
+        self,
+        text: str,
+        *,
+        num_results: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        snippet_length: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Search the web via the Diffbot LLM web search API.
+
+        snippet_length: "short", "medium", "long" or "full" — how much of each
+        page's matching content to return. Omit for the API default (long).
+        "full" returns the whole stored page (reranking is disabled for it);
+        use it with a url:<URL> query.
+        """
+        return _web_search(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
 
     def entities(self, text: str, *, lang: str = "auto") -> Dict[str, Any]:
         """Identify and resolve entities and sentiment in text using the Diffbot NLP API.
@@ -349,9 +362,22 @@ class DiffbotAsync:
         """Download the ontology and return it as a queryable Ontology (no caching)."""
         return await _dql_fetch_ontology_async(self)
 
-    async def web_search(self, text: str, *, num_results: Optional[int] = None, max_tokens: Optional[int] = None) -> Dict[str, Any]:
-        """Search the web via the Diffbot LLM web search API."""
-        return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens)
+    async def web_search(
+        self,
+        text: str,
+        *,
+        num_results: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        snippet_length: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Search the web via the Diffbot LLM web search API.
+
+        snippet_length: "short", "medium", "long" or "full" — how much of each
+        page's matching content to return. Omit for the API default (long).
+        "full" returns the whole stored page (reranking is disabled for it);
+        use it with a url:<URL> query.
+        """
+        return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens, snippet_length=snippet_length)
 
     async def entities(self, text: str, *, lang: str = "auto") -> Dict[str, Any]:
         """Identify and resolve entities and sentiment in text using the Diffbot NLP API.

{diffbot-3.0.0 → diffbot-3.1.0}/src/diffbot/web_search.py

@@ -14,6 +14,7 @@ def web_search(
     *,
     num_results: Optional[int] = None,
     max_tokens: Optional[int] = None,
+    snippet_length: Optional[str] = None,
 ) -> Dict[str, Any]:
     headers = {"Authorization": f"Bearer {client.token}"}
     params: Dict[str, Any] = {"text": text}
@@ -21,6 +22,11 @@ def web_search(
         params["size"] = num_results
     if max_tokens is not None:
         params["maxTokens"] = max_tokens
+    if snippet_length is not None:
+        params["snippetLength"] = snippet_length.upper()
+        if params["snippetLength"] == "FULL":
+            # the reranker always chunks; only the unranked path returns the stored page
+            params["rerank"] = "none"
     response = client._http.get(client.web_search_url, headers=headers, params=params)
     client._raise_for_status(response)
     return response.json()
@@ -32,6 +38,7 @@ async def web_search_async(
     *,
     num_results: Optional[int] = None,
     max_tokens: Optional[int] = None,
+    snippet_length: Optional[str] = None,
 ) -> Dict[str, Any]:
     headers = {"Authorization": f"Bearer {client.token}"}
     params: Dict[str, Any] = {"text": text}
@@ -39,6 +46,11 @@ async def web_search_async(
         params["size"] = num_results
     if max_tokens is not None:
         params["maxTokens"] = max_tokens
+    if snippet_length is not None:
+        params["snippetLength"] = snippet_length.upper()
+        if params["snippetLength"] == "FULL":
+            # the reranker always chunks; only the unranked path returns the stored page
+            params["rerank"] = "none"
     response = await client._http.get(client.web_search_url, headers=headers, params=params)
     client._raise_for_status(response)
     return response.json()

diffbot-3.1.0/tests/test_web_search.py

@@ -0,0 +1,51 @@
+import pytest
+
+
+"""
+Live
+"""
+
+@pytest.mark.live
+def test_live_web_search(db):
+    result = db.web_search("diffbot knowledge graph", num_results=3)
+    assert "search_results" in result
+    assert len(result["search_results"]) > 0
+
+
+@pytest.mark.live
+def test_live_web_search_snippet_length(db):
+    long = db.web_search("diffbot knowledge graph", num_results=3)
+    short = db.web_search("diffbot knowledge graph", num_results=3, snippet_length="short")
+    assert len(short["search_results"]) > 0
+    total_long = sum(len(r.get("content", "")) for r in long["search_results"])
+    total_short = sum(len(r.get("content", "")) for r in short["search_results"])
+    assert total_short <= total_long
+
+
+"""
+Mocked
+"""
+
+def test_web_search_sends_snippet_length_param():
+    import httpx
+    from diffbot import Diffbot
+
+    seen = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen.update(request.url.params)
+        return httpx.Response(200, json={"search_results": []})
+
+    db = Diffbot(token="t", transport=httpx.MockTransport(handler))
+    db.web_search("q", num_results=3, snippet_length="short")
+    assert seen["size"] == "3"
+    assert seen["snippetLength"] == "SHORT"
+
+    seen.clear()
+    db.web_search("q")
+    assert "snippetLength" not in seen
+
+    seen.clear()
+    db.web_search("url:https://example.com", snippet_length="full")
+    assert seen["snippetLength"] == "FULL"
+    assert seen["rerank"] == "none"

diffbot-3.0.0/tests/test_web_search.py

@@ -1,12 +0,0 @@
-import pytest
-
-
-"""
-Live
-"""
-
-@pytest.mark.live
-def test_live_web_search(db):
-    result = db.web_search("diffbot knowledge graph", num_results=3)
-    assert "search_results" in result
-    assert len(result["search_results"]) > 0