diffbot-python 0.1.0__py3-none-any.whl

diffbot/kg.py

@@ -0,0 +1,90 @@
+"""Diffbot Knowledge Graph APIs: DQL search and entity enhancement."""
+
+import pathlib
+from concurrent.futures import ThreadPoolExecutor
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+KG_DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql"
+KG_ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology"
+
+
+def _build_dql_params(
+    client: Any,
+    query: str,
+    size: int,
+    from_: int,
+    format: str,
+    filter: Optional[str],
+    exportspec: Optional[str],
+    extra: Optional[Dict[str, str]],
+) -> Dict[str, Any]:
+    params: Dict[str, Any] = {"token": client.token, "query": query, "size": size}
+    if from_:
+        params["from"] = from_
+    if format != "json":
+        params["format"] = format
+    if filter is not None:
+        params["filter"] = filter
+    if exportspec is not None:
+        params["exportspec"] = exportspec
+    if extra:
+        params.update(extra)
+    return params
+
+
+def dql(
+    client: "Diffbot",
+    query: str,
+    *,
+    size: int = 10,
+    from_: int = 0,
+    format: str = "json",
+    filter: Optional[str] = None,
+    exportspec: Optional[str] = None,
+    extra: Optional[Dict[str, str]] = None,
+    raw: bool = False,
+) -> Union[Dict[str, Any], bytes]:
+    params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra)
+    response = client._http.get(KG_DQL_ENDPOINT, params=params)
+    client._raise_for_status(response)
+    return response.content if raw else response.json()
+
+
+async def dql_async(
+    client: "DiffbotAsync",
+    query: str,
+    *,
+    size: int = 10,
+    from_: int = 0,
+    format: str = "json",
+    filter: Optional[str] = None,
+    exportspec: Optional[str] = None,
+    extra: Optional[Dict[str, str]] = None,
+    raw: bool = False,
+) -> Union[Dict[str, Any], bytes]:
+    params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra)
+    response = await client._http.get(KG_DQL_ENDPOINT, params=params)
+    client._raise_for_status(response)
+    return response.content if raw else response.json()
+
+
+def dql_parallel(
+    client: "Diffbot",
+    queries: Sequence[Dict[str, Any]],
+    *,
+    workers: int = 8,
+) -> List[Union[Dict[str, Any], bytes]]:
+    if not queries:
+        return []
+    with ThreadPoolExecutor(max_workers=min(workers, len(queries))) as ex:
+        return list(ex.map(lambda q: dql(client, **q), queries))
+
+
+def dql_refresh_ontology(client: "Diffbot", dest: pathlib.Path) -> None:
+    response = client._http.get(KG_ONTOLOGY_ENDPOINT)
+    client._raise_for_status(response)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    dest.write_bytes(response.content)

diffbot/nlp.py

@@ -0,0 +1,37 @@
+"""Diffbot NLP API: entity identification, resolution, and sentiment."""
+
+from typing import TYPE_CHECKING, Any, Dict
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+NLP_BASE = "https://nl.diffbot.com/v1/"
+NLP_FIELDS = "entities,sentiment"
+
+
+def entities(
+    client: "Diffbot",
+    text: str,
+    *,
+    lang: str = "auto",
+) -> Dict[str, Any]:
+    params = {"token": client.token, "fields": NLP_FIELDS}
+    payload = [{"lang": lang, "format": "plain text", "content": text}]
+    response = client._http.post(client.nlp_url, params=params, json=payload)
+    client._raise_for_status(response)
+    data = response.json()
+    return data[0] if isinstance(data, list) else data
+
+
+async def entities_async(
+    client: "DiffbotAsync",
+    text: str,
+    *,
+    lang: str = "auto",
+) -> Dict[str, Any]:
+    params = {"token": client.token, "fields": NLP_FIELDS}
+    payload = [{"lang": lang, "format": "plain text", "content": text}]
+    response = await client._http.post(client.nlp_url, params=params, json=payload)
+    client._raise_for_status(response)
+    data = response.json()
+    return data[0] if isinstance(data, list) else data

diffbot/web_search.py

@@ -0,0 +1,44 @@
+"""Diffbot web search API."""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+WEB_SEARCH_BASE = "https://llm.diffbot.com/api/v1/web_search"
+
+
+def web_search(
+    client: "Diffbot",
+    text: str,
+    *,
+    num_results: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+) -> Dict[str, Any]:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    params: Dict[str, Any] = {"text": text}
+    if num_results is not None:
+        params["num_results"] = num_results
+    if max_tokens is not None:
+        params["maxTokens"] = max_tokens
+    response = client._http.get(client.web_search_url, headers=headers, params=params)
+    client._raise_for_status(response)
+    return response.json()
+
+
+async def web_search_async(
+    client: "DiffbotAsync",
+    text: str,
+    *,
+    num_results: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+) -> Dict[str, Any]:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    params: Dict[str, Any] = {"text": text}
+    if num_results is not None:
+        params["num_results"] = num_results
+    if max_tokens is not None:
+        params["maxTokens"] = max_tokens
+    response = await client._http.get(client.web_search_url, headers=headers, params=params)
+    client._raise_for_status(response)
+    return response.json()

diffbot_python-0.1.0.dist-info/METADATA

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: diffbot-python
+Version: 0.1.0
+Summary: Python client library for Diffbot APIs
+Project-URL: Homepage, https://github.com/diffbot/diffbot-python
+Project-URL: Repository, https://github.com/diffbot/diffbot-python
+Project-URL: Issues, https://github.com/diffbot/diffbot-python/issues
+Author-email: Jerome Choo <jerome@diffbot.com>, Mike Tung <miket@diffbot.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: rich>=13.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Diffbot Python Library
+
+Python client library for [Diffbot](https://www.diffbot.com) APIs.
+
+
+## Installation
+
+```bash
+pip install git+https://github.com/diffbot/diffbot-python.git
+```
+
+Or, for local development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Usage
+
+### Authentication
+Set your Diffbot API token in your environment or .env.
+
+```bash
+export DIFFBOT_API_TOKEN=<TOKEN>
+```
+
+### Extract structured content
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+data = db.extract("https://www.example.com")
+```
+
+### Ask Diffbot LLM
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+for chunk in db.ask([{"role": "user", "content": "What's the capital of France?"}]):
+    print(chunk, end="")
+```
+
+### Crawl a site for structured content
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+for event in db.crawl("https://www.example.com", hops=1):
+    print(event)
+```
+
+### Query the Knowledge Graph
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+results = db.dql('type:Organization name:"Diffbot"')
+```
+
+### Web Search
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+results = db.web_search("diffbot knowledge graph")
+for r in results["search_results"]:
+    print(r["score"], r["title"], r["pageUrl"])
+    print(r["content"])
+```
+
+### Entities (NLP)
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+result = db.entities("Apple CEO Tim Cook announced record quarterly earnings.")
+for entity in result["entities"]:
+    print(entity["name"], entity.get("type"), entity.get("id"))
+print("sentiment:", result.get("sentiment"))
+```
+
+## Async Usage
+
+### Extract structured content
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        data = await db.extract("https://www.example.com")
+        print(data)
+
+asyncio.run(main())
+```
+
+### Ask Diffbot LLM
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        async for chunk in db.ask([{"role": "user", "content": "What's the capital of France?"}]):
+            print(chunk, end="")
+
+asyncio.run(main())
+```
+
+### Crawl a site for structured content
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        async for event in db.crawl("https://www.example.com", hops=1):
+            print(event)
+
+asyncio.run(main())
+```
+
+### Query the Knowledge Graph
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        results = await db.dql('type:Organization name:"Diffbot"')
+        print(results)
+
+asyncio.run(main())
+```
+
+### Web Search
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        results = await db.web_search("diffbot knowledge graph")
+        for r in results["search_results"]:
+            print(r["score"], r["title"], r["pageUrl"])
+            print(r["content"])
+
+asyncio.run(main())
+```
+
+### Entities (NLP)
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        result = await db.entities("Apple CEO Tim Cook announced record quarterly earnings.")
+        for entity in result["entities"]:
+            print(entity["name"], entity.get("type"), entity.get("id"))
+        print("sentiment:", result.get("sentiment"))
+
+asyncio.run(main())
+```
+
+## CLI
+
+This library also includes a CLI.
+
+```bash
+export DIFFBOT_API_TOKEN=your-token-here
+
+db extract https://www.example.com
+db ask "What's the capital of France?"
+db crawl https://www.example.com --hops 1
+db crawl-list-jobs
+db crawl-delete-job crawl-1234567890
+db web-search "diffbot knowledge graph"
+db web-search "diffbot knowledge graph" -n 5 -f json
+db entities "Apple CEO Tim Cook announced record quarterly earnings."
+db entities "Apple CEO Tim Cook announced record quarterly earnings." -f dql
+```
+
+## Tests
+
+Run the mock test suite:
+```bash
+python -m pytest
+```
+
+Run live integration tests against the real API (requires a valid token):
+```bash
+DIFFBOT_TOKEN=your_token python -m pytest -m live
+```

diffbot_python-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+diffbot/__init__.py,sha256=WyzW2kAw75vgEmAyvrFkfcu_m0gL8uhOixeYVCKaHf4,518
+diffbot/ask.py,sha256=iNv613j4CoIfdDTOE-pl9KUkjqI-2AxGDMR1prm3DGM,1853
+diffbot/client.py,sha256=C5MIhyx8YZGgD4GjKj5QrIg-eRtUF3qhmUzYW77Sh9Q,10426
+diffbot/crawl.py,sha256=iYMFmf7HKrbefJrGg14VnlfiBFLOE_Z1pfO4Rn_cDXc,8893
+diffbot/errors.py,sha256=5-AceX5MyNVUhe9pvR_4rnQQmBhvLfwWmrRl7dRZUSg,1576
+diffbot/extract.py,sha256=R9SVxaOi4FjHOQIX5ho_75OwaJ7VdX-mdAoV_UY-lrM,1452
+diffbot/kg.py,sha256=Y7XTrPNAfPdX9vvhFmgmU4G4KTF9fwpYkm2Hh3c6DLA,2708
+diffbot/nlp.py,sha256=lZJW4MkjhVklIEM2OBfhc8LvresXTy7RsuImPteYsOA,1153
+diffbot/web_search.py,sha256=1sKBojzsslZj2zzl2kJ4s43AUaRn1i1fGgPUjFdsW6Q,1370
+diffbot/cli/__init__.py,sha256=UVGD3uevKTHmqEdbDhhR2PzO6-3i0xu8d7D_94jLLRo,16488
+diffbot/cli/__main__.py,sha256=5BjNuyet8AY-POwoF5rGt722rHQ7tJ0Vf0UFUfzzi-I,58
+diffbot/cli/_common.py,sha256=0I-oHnKVM9zLCaVNDc-7qf17b5u827IK7ezXITjsdW4,1006
+diffbot/cli/dql.py,sha256=lJzAEjTIoF1l1xTwMcVIlq36pKZKrvgZYz0RuX5jUGc,11419
+diffbot/cli/entities.py,sha256=tsHKexF0b6NnsoUEZJUS2rJidTIE8lt6xrcSenPbwtY,5630
+diffbot/cli/ontology.py,sha256=FLIIe6ZY34zLHLt_bB9Zci0zlrKEAIIgNSdz5KwXqzw,3773
+diffbot_python-0.1.0.dist-info/METADATA,sha256=KrMqxyqa2g6GPee1zc8A6J1SPXDLukELFslwbuMj9CM,5281
+diffbot_python-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+diffbot_python-0.1.0.dist-info/entry_points.txt,sha256=FCxJhrbl7VNEsTK7zl7qYvXID7gQ1_wxhiw5_Vllb_M,40
+diffbot_python-0.1.0.dist-info/licenses/LICENSE,sha256=UZlamI1XGeiG0Mit8dsHssNhOuMGKfmNOp5qpf1533w,1063
+diffbot_python-0.1.0.dist-info/RECORD,,

diffbot_python-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

diffbot_python-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+db = diffbot.cli:main

diffbot_python-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Diffbot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.