diffbot-python 0.1.0__tar.gz

diffbot_python-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+# Python
+.venv
+__pycache__
+.pytest_cache
+.env
+
+# Claude
+.claude/settings.local.json

diffbot_python-0.1.0/AGENTS.md

@@ -0,0 +1,5 @@
+# Agent Guidelines
+
+## README Examples
+
+Whenever a code example in `README.md` is added or updated, the corresponding test must be added or updated in `tests/test_readme_examples.py`. Run `python -m pytest tests/test_readme_examples.py` to validate before considering the work complete.

diffbot_python-0.1.0/CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

diffbot_python-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Diffbot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

diffbot_python-0.1.0/PKG-INFO

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: diffbot-python
+Version: 0.1.0
+Summary: Python client library for Diffbot APIs
+Project-URL: Homepage, https://github.com/diffbot/diffbot-python
+Project-URL: Repository, https://github.com/diffbot/diffbot-python
+Project-URL: Issues, https://github.com/diffbot/diffbot-python/issues
+Author-email: Jerome Choo <jerome@diffbot.com>, Mike Tung <miket@diffbot.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: rich>=13.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Diffbot Python Library
+
+Python client library for [Diffbot](https://www.diffbot.com) APIs.
+
+
+## Installation
+
+```bash
+pip install git+https://github.com/diffbot/diffbot-python.git
+```
+
+Or, for local development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Usage
+
+### Authentication
+Set your Diffbot API token in your environment or .env.
+
+```bash
+export DIFFBOT_API_TOKEN=<TOKEN>
+```
+
+### Extract structured content
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+data = db.extract("https://www.example.com")
+```
+
+### Ask Diffbot LLM
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+for chunk in db.ask([{"role": "user", "content": "What's the capital of France?"}]):
+    print(chunk, end="")
+```
+
+### Crawl a site for structured content
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+for event in db.crawl("https://www.example.com", hops=1):
+    print(event)
+```
+
+### Query the Knowledge Graph
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+results = db.dql('type:Organization name:"Diffbot"')
+```
+
+### Web Search
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+results = db.web_search("diffbot knowledge graph")
+for r in results["search_results"]:
+    print(r["score"], r["title"], r["pageUrl"])
+    print(r["content"])
+```
+
+### Entities (NLP)
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+result = db.entities("Apple CEO Tim Cook announced record quarterly earnings.")
+for entity in result["entities"]:
+    print(entity["name"], entity.get("type"), entity.get("id"))
+print("sentiment:", result.get("sentiment"))
+```
+
+## Async Usage
+
+### Extract structured content
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        data = await db.extract("https://www.example.com")
+        print(data)
+
+asyncio.run(main())
+```
+
+### Ask Diffbot LLM
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        async for chunk in db.ask([{"role": "user", "content": "What's the capital of France?"}]):
+            print(chunk, end="")
+
+asyncio.run(main())
+```
+
+### Crawl a site for structured content
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        async for event in db.crawl("https://www.example.com", hops=1):
+            print(event)
+
+asyncio.run(main())
+```
+
+### Query the Knowledge Graph
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        results = await db.dql('type:Organization name:"Diffbot"')
+        print(results)
+
+asyncio.run(main())
+```
+
+### Web Search
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        results = await db.web_search("diffbot knowledge graph")
+        for r in results["search_results"]:
+            print(r["score"], r["title"], r["pageUrl"])
+            print(r["content"])
+
+asyncio.run(main())
+```
+
+### Entities (NLP)
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        result = await db.entities("Apple CEO Tim Cook announced record quarterly earnings.")
+        for entity in result["entities"]:
+            print(entity["name"], entity.get("type"), entity.get("id"))
+        print("sentiment:", result.get("sentiment"))
+
+asyncio.run(main())
+```
+
+## CLI
+
+This library also includes a CLI.
+
+```bash
+export DIFFBOT_API_TOKEN=your-token-here
+
+db extract https://www.example.com
+db ask "What's the capital of France?"
+db crawl https://www.example.com --hops 1
+db crawl-list-jobs
+db crawl-delete-job crawl-1234567890
+db web-search "diffbot knowledge graph"
+db web-search "diffbot knowledge graph" -n 5 -f json
+db entities "Apple CEO Tim Cook announced record quarterly earnings."
+db entities "Apple CEO Tim Cook announced record quarterly earnings." -f dql
+```
+
+## Tests
+
+Run the mock test suite:
+```bash
+python -m pytest
+```
+
+Run live integration tests against the real API (requires a valid token):
+```bash
+DIFFBOT_TOKEN=your_token python -m pytest -m live
+```

diffbot_python-0.1.0/README.md

@@ -0,0 +1,195 @@
+# Diffbot Python Library
+
+Python client library for [Diffbot](https://www.diffbot.com) APIs.
+
+
+## Installation
+
+```bash
+pip install git+https://github.com/diffbot/diffbot-python.git
+```
+
+Or, for local development:
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Usage
+
+### Authentication
+Set your Diffbot API token in your environment or .env.
+
+```bash
+export DIFFBOT_API_TOKEN=<TOKEN>
+```
+
+### Extract structured content
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+data = db.extract("https://www.example.com")
+```
+
+### Ask Diffbot LLM
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+for chunk in db.ask([{"role": "user", "content": "What's the capital of France?"}]):
+    print(chunk, end="")
+```
+
+### Crawl a site for structured content
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+for event in db.crawl("https://www.example.com", hops=1):
+    print(event)
+```
+
+### Query the Knowledge Graph
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+results = db.dql('type:Organization name:"Diffbot"')
+```
+
+### Web Search
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+results = db.web_search("diffbot knowledge graph")
+for r in results["search_results"]:
+    print(r["score"], r["title"], r["pageUrl"])
+    print(r["content"])
+```
+
+### Entities (NLP)
+```python
+from diffbot import Diffbot
+
+db = Diffbot(token="YOUR_TOKEN")
+result = db.entities("Apple CEO Tim Cook announced record quarterly earnings.")
+for entity in result["entities"]:
+    print(entity["name"], entity.get("type"), entity.get("id"))
+print("sentiment:", result.get("sentiment"))
+```
+
+## Async Usage
+
+### Extract structured content
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        data = await db.extract("https://www.example.com")
+        print(data)
+
+asyncio.run(main())
+```
+
+### Ask Diffbot LLM
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        async for chunk in db.ask([{"role": "user", "content": "What's the capital of France?"}]):
+            print(chunk, end="")
+
+asyncio.run(main())
+```
+
+### Crawl a site for structured content
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        async for event in db.crawl("https://www.example.com", hops=1):
+            print(event)
+
+asyncio.run(main())
+```
+
+### Query the Knowledge Graph
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        results = await db.dql('type:Organization name:"Diffbot"')
+        print(results)
+
+asyncio.run(main())
+```
+
+### Web Search
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        results = await db.web_search("diffbot knowledge graph")
+        for r in results["search_results"]:
+            print(r["score"], r["title"], r["pageUrl"])
+            print(r["content"])
+
+asyncio.run(main())
+```
+
+### Entities (NLP)
+```python
+import asyncio
+from diffbot import DiffbotAsync
+
+async def main():
+    async with DiffbotAsync(token="YOUR_TOKEN") as db:
+        result = await db.entities("Apple CEO Tim Cook announced record quarterly earnings.")
+        for entity in result["entities"]:
+            print(entity["name"], entity.get("type"), entity.get("id"))
+        print("sentiment:", result.get("sentiment"))
+
+asyncio.run(main())
+```
+
+## CLI
+
+This library also includes a CLI.
+
+```bash
+export DIFFBOT_API_TOKEN=your-token-here
+
+db extract https://www.example.com
+db ask "What's the capital of France?"
+db crawl https://www.example.com --hops 1
+db crawl-list-jobs
+db crawl-delete-job crawl-1234567890
+db web-search "diffbot knowledge graph"
+db web-search "diffbot knowledge graph" -n 5 -f json
+db entities "Apple CEO Tim Cook announced record quarterly earnings."
+db entities "Apple CEO Tim Cook announced record quarterly earnings." -f dql
+```
+
+## Tests
+
+Run the mock test suite:
+```bash
+python -m pytest
+```
+
+Run live integration tests against the real API (requires a valid token):
+```bash
+DIFFBOT_TOKEN=your_token python -m pytest -m live
+```

diffbot_python-0.1.0/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "diffbot-python"
+version = "0.1.0"
+description = "Python client library for Diffbot APIs"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search"
+]
+license = "MIT"
+license-files = ["LICEN[CS]E*"]
+authors = [
+    { name = "Jerome Choo", email = "jerome@diffbot.com" },
+    { name = "Mike Tung", email = "miket@diffbot.com" }
+]
+dependencies = [
+    "httpx>=0.27.0",
+    "click>=8.1.0",
+    "rich>=13.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/diffbot/diffbot-python"
+Repository = "https://github.com/diffbot/diffbot-python"
+Issues = "https://github.com/diffbot/diffbot-python/issues"
+
+[project.scripts]
+db = "diffbot.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/diffbot"]
+
+[tool.pytest.ini_options]
+markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_TOKEN"]
+addopts = "-m 'not live'"

diffbot_python-0.1.0/src/diffbot/__init__.py

@@ -0,0 +1,29 @@
+"""
+diffbot - Python client library for the Diffbot APIs.
+"""
+
+__version__ = "0.1.0"
+
+from .client import Diffbot, DiffbotAsync
+from .crawl import CrawlEvent, CrawlEventType
+from .errors import (
+    APIError,
+    AuthError,
+    DiffbotError,
+    ExtractionError,
+    RateLimitError,
+    ValidationError,
+)
+
+__all__ = [
+    "Diffbot",
+    "DiffbotAsync",
+    "CrawlEvent",
+    "CrawlEventType",
+    "DiffbotError",
+    "AuthError",
+    "ExtractionError",
+    "RateLimitError",
+    "APIError",
+    "ValidationError",
+]

diffbot_python-0.1.0/src/diffbot/ask.py

@@ -0,0 +1,48 @@
+"""Diffbot LLM RAG API: stream a chat completion."""
+
+import json
+from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+
+def _build_payload(client: Any, messages: List[Dict[str, str]]) -> tuple:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    payload = {"model": "diffbot-small-xl", "messages": messages, "stream": True}
+    return headers, payload
+
+
+def _parse_chunk(line: str):
+    try:
+        chunk = json.loads(line.replace("data: ", ""))
+    except json.JSONDecodeError:
+        return None
+    choices = chunk.get("choices")
+    if choices and choices[0].get("delta", {}).get("content"):
+        return choices[0]["delta"]["content"]
+    return None
+
+
+def ask(client: "Diffbot", messages: List[Dict[str, str]]) -> Iterator[str]:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    payload = {"model": "diffbot-small-xl", "messages": messages, "stream": True}
+    with client._http.stream("POST", client.llm_url, headers=headers, json=payload) as response:
+        client._raise_for_status(response)
+        for line in response.iter_lines():
+            if line:
+                content = _parse_chunk(line)
+                if content:
+                    yield content
+
+
+async def ask_async(client: "DiffbotAsync", messages: List[Dict[str, str]]) -> AsyncIterator[str]:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    payload = {"model": "diffbot-small-xl", "messages": messages, "stream": True}
+    async with client._http.stream("POST", client.llm_url, headers=headers, json=payload) as response:
+        client._raise_for_status(response)
+        async for line in response.aiter_lines():
+            if line:
+                content = _parse_chunk(line)
+                if content:
+                    yield content