diffbot-python 0.1.0__tar.gz → 0.2.0__tar.gz

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/.gitignore

@@ -3,6 +3,9 @@
 __pycache__
 .pytest_cache
 .env
+dist/
+build/
+*.egg-info/
 
 # Claude
 .claude/settings.local.json

diffbot_python-0.1.0/README.md → diffbot_python-0.2.0/PKG-INFO

@@ -1,3 +1,38 @@
+Metadata-Version: 2.4
+Name: diffbot-python
+Version: 0.2.0
+Summary: Python client library for Diffbot APIs
+Project-URL: Homepage, https://github.com/diffbot/diffbot-python
+Project-URL: Documentation, https://github.com/diffbot/diffbot-python#readme
+Project-URL: Repository, https://github.com/diffbot/diffbot-python
+Project-URL: Issues, https://github.com/diffbot/diffbot-python/issues
+Author-email: Jerome Choo <jerome@diffbot.com>, Mike Tung <miket@diffbot.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: api-client,crawler,diffbot,extract,knowledge-graph,llm,nlp,web-scraping
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: rich>=13.0.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
 # Diffbot Python Library
 
 Python client library for [Diffbot](https://www.diffbot.com) APIs.
@@ -6,7 +41,7 @@ Python client library for [Diffbot](https://www.diffbot.com) APIs.
 ## Installation
 
 ```bash
-pip install git+https://github.com/diffbot/diffbot-python.git
+python3 -m pip install diffbot-python
 ```
 
 Or, for local development:
@@ -18,12 +53,38 @@ pip install -e ".[dev]"
 ## Usage
 
 ### Authentication
-Set your Diffbot API token in your environment or .env.
+
+The CLI and the library can share a single credential. The token always has to be
+passed to the client explicitly, but `resolve_token()` gives you the same lookup the
+CLI uses, in this order:
+
+1. An explicit token passed to `resolve_token(token)`.
+2. The `DIFFBOT_API_TOKEN` environment variable.
+3. A `DIFFBOT_API_TOKEN=...` line in `~/.diffbot/credentials`.
+
+Set it once and it works for both the CLI and your scripts. Either export it:
 
 ```bash
 export DIFFBOT_API_TOKEN=<TOKEN>
 ```
 
+…or write it to the shared credentials file (handy for keeping it out of your shell environment):
+
+```bash
+mkdir -p ~/.diffbot
+printf 'DIFFBOT_API_TOKEN=%s\n' '<TOKEN>' > ~/.diffbot/credentials
+chmod 600 ~/.diffbot/credentials
+```
+
+With either in place, resolve the token and pass it to the client:
+
+```python
+from diffbot import Diffbot, resolve_token
+
+db = Diffbot(token=resolve_token())  # from env var or ~/.diffbot/credentials
+data = db.extract("https://www.example.com")
+```
+
 ### Extract structured content
 ```python
 from diffbot import Diffbot
@@ -166,7 +227,15 @@ asyncio.run(main())
 
 ## CLI
 
-This library also includes a CLI.
+This library also includes a CLI exposed as the `db` command.
+
+To make `db` available from anywhere, install it as an isolated tool with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv tool install .
+```
+
+This drops a `db` executable into `~/.local/bin` (ensure it is on your `PATH`). Use `--force` to reinstall or upgrade after changes, or `--editable` to have source edits take effect immediately. Alternatively, a plain `pip install .` (or `pip install -e .`) also installs the `db` entry point into the active environment.
 
 ```bash
 export DIFFBOT_API_TOKEN=your-token-here
@@ -189,7 +258,9 @@ Run the mock test suite:
 python -m pytest
 ```
 
-Run live integration tests against the real API (requires a valid token):
+Run live integration tests against the real API (requires a valid token).
+The token is resolved the same way as everywhere else — the `DIFFBOT_API_TOKEN`
+environment variable or `~/.diffbot/credentials`:
 ```bash
-DIFFBOT_TOKEN=your_token python -m pytest -m live
+DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```

diffbot_python-0.1.0/PKG-INFO → diffbot_python-0.2.0/README.md

@@ -1,26 +1,3 @@
-Metadata-Version: 2.4
-Name: diffbot-python
-Version: 0.1.0
-Summary: Python client library for Diffbot APIs
-Project-URL: Homepage, https://github.com/diffbot/diffbot-python
-Project-URL: Repository, https://github.com/diffbot/diffbot-python
-Project-URL: Issues, https://github.com/diffbot/diffbot-python/issues
-Author-email: Jerome Choo <jerome@diffbot.com>, Mike Tung <miket@diffbot.com>
-License-Expression: MIT
-License-File: LICENSE
-Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Software Development :: Libraries
-Requires-Python: >=3.10
-Requires-Dist: click>=8.1.0
-Requires-Dist: httpx>=0.27.0
-Requires-Dist: rich>=13.0.0
-Provides-Extra: dev
-Requires-Dist: pytest>=8.0.0; extra == 'dev'
-Description-Content-Type: text/markdown
-
 # Diffbot Python Library
 
 Python client library for [Diffbot](https://www.diffbot.com) APIs.
@@ -29,7 +6,7 @@ Python client library for [Diffbot](https://www.diffbot.com) APIs.
 ## Installation
 
 ```bash
-pip install git+https://github.com/diffbot/diffbot-python.git
+python3 -m pip install diffbot-python
 ```
 
 Or, for local development:
@@ -41,12 +18,38 @@ pip install -e ".[dev]"
 ## Usage
 
 ### Authentication
-Set your Diffbot API token in your environment or .env.
+
+The CLI and the library can share a single credential. The token always has to be
+passed to the client explicitly, but `resolve_token()` gives you the same lookup the
+CLI uses, in this order:
+
+1. An explicit token passed to `resolve_token(token)`.
+2. The `DIFFBOT_API_TOKEN` environment variable.
+3. A `DIFFBOT_API_TOKEN=...` line in `~/.diffbot/credentials`.
+
+Set it once and it works for both the CLI and your scripts. Either export it:
 
 ```bash
 export DIFFBOT_API_TOKEN=<TOKEN>
 ```
 
+…or write it to the shared credentials file (handy for keeping it out of your shell environment):
+
+```bash
+mkdir -p ~/.diffbot
+printf 'DIFFBOT_API_TOKEN=%s\n' '<TOKEN>' > ~/.diffbot/credentials
+chmod 600 ~/.diffbot/credentials
+```
+
+With either in place, resolve the token and pass it to the client:
+
+```python
+from diffbot import Diffbot, resolve_token
+
+db = Diffbot(token=resolve_token())  # from env var or ~/.diffbot/credentials
+data = db.extract("https://www.example.com")
+```
+
 ### Extract structured content
 ```python
 from diffbot import Diffbot
@@ -189,7 +192,15 @@ asyncio.run(main())
 
 ## CLI
 
-This library also includes a CLI.
+This library also includes a CLI exposed as the `db` command.
+
+To make `db` available from anywhere, install it as an isolated tool with [uv](https://docs.astral.sh/uv/):
+
+```bash
+uv tool install .
+```
+
+This drops a `db` executable into `~/.local/bin` (ensure it is on your `PATH`). Use `--force` to reinstall or upgrade after changes, or `--editable` to have source edits take effect immediately. Alternatively, a plain `pip install .` (or `pip install -e .`) also installs the `db` entry point into the active environment.
 
 ```bash
 export DIFFBOT_API_TOKEN=your-token-here
@@ -212,7 +223,9 @@ Run the mock test suite:
 python -m pytest
 ```
 
-Run live integration tests against the real API (requires a valid token):
+Run live integration tests against the real API (requires a valid token).
+The token is resolved the same way as everywhere else — the `DIFFBOT_API_TOKEN`
+environment variable or `~/.diffbot/credentials`:
 ```bash
-DIFFBOT_TOKEN=your_token python -m pytest -m live
+DIFFBOT_API_TOKEN=your_token python -m pytest -m live
 ```

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/pyproject.toml

@@ -4,23 +4,43 @@ build-backend = "hatchling.build"
 
 [project]
 name = "diffbot-python"
-version = "0.1.0"
+version = "0.2.0"
 description = "Python client library for Diffbot APIs"
 readme = "README.md"
 requires-python = ">=3.10"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "Operating System :: OS Independent",
-    "Topic :: Software Development :: Libraries",
-    "Topic :: Scientific/Engineering :: Artificial Intelligence",
-    "Topic :: Internet :: WWW/HTTP :: Indexing/Search"
-]
 license = "MIT"
 license-files = ["LICEN[CS]E*"]
 authors = [
     { name = "Jerome Choo", email = "jerome@diffbot.com" },
     { name = "Mike Tung", email = "miket@diffbot.com" }
 ]
+keywords = [
+    "diffbot",
+    "knowledge-graph",
+    "web-scraping",
+    "extract",
+    "crawler",
+    "nlp",
+    "llm",
+    "api-client",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Typing :: Typed",
+]
 dependencies = [
     "httpx>=0.27.0",
     "click>=8.1.0",
@@ -34,6 +54,7 @@ dev = [
 
 [project.urls]
 Homepage = "https://github.com/diffbot/diffbot-python"
+Documentation = "https://github.com/diffbot/diffbot-python#readme"
 Repository = "https://github.com/diffbot/diffbot-python"
 Issues = "https://github.com/diffbot/diffbot-python/issues"
 
@@ -43,6 +64,15 @@ db = "diffbot.cli:main"
 [tool.hatch.build.targets.wheel]
 packages = ["src/diffbot"]
 
+[tool.hatch.build.targets.sdist]
+include = [
+    "/src",
+    "/tests",
+    "/README.md",
+    "/LICENSE",
+    "/pyproject.toml",
+]
+
 [tool.pytest.ini_options]
-markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_TOKEN"]
-addopts = "-m 'not live'"
+markers = ["live: marks tests as live integration tests requiring a real DIFFBOT_API_TOKEN"]
+addopts = "-m 'not live'"

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/src/diffbot/__init__.py

@@ -4,6 +4,7 @@ diffbot - Python client library for the Diffbot APIs.
 
 __version__ = "0.1.0"
 
+from ._auth import resolve_token
 from .client import Diffbot, DiffbotAsync
 from .crawl import CrawlEvent, CrawlEventType
 from .errors import (
@@ -14,12 +15,15 @@ from .errors import (
     RateLimitError,
     ValidationError,
 )
+from .ontology import Ontology
 
 __all__ = [
     "Diffbot",
     "DiffbotAsync",
+    "resolve_token",
     "CrawlEvent",
     "CrawlEventType",
+    "Ontology",
     "DiffbotError",
     "AuthError",
     "ExtractionError",

diffbot_python-0.2.0/src/diffbot/_auth.py

@@ -0,0 +1,41 @@
+"""Shared Diffbot credential resolution for both the library and the CLI.
+
+The same lookup chain is used everywhere so a single credential works for the
+``db`` CLI and any Python script that constructs a client:
+
+1. An explicit token passed to the client / function.
+2. The ``DIFFBOT_API_TOKEN`` environment variable.
+3. A ``DIFFBOT_API_TOKEN=...`` line in ``~/.diffbot/credentials``.
+"""
+
+import os
+import pathlib
+from typing import Optional
+
+TOKEN_ENV_VAR = "DIFFBOT_API_TOKEN"
+CREDENTIALS_PATH = pathlib.Path.home() / ".diffbot" / "credentials"
+
+
+def _read_credentials_file() -> str:
+    if not CREDENTIALS_PATH.exists():
+        return ""
+    for line in CREDENTIALS_PATH.read_text().splitlines():
+        line = line.strip()
+        if line.startswith(f"{TOKEN_ENV_VAR}="):
+            return line[len(TOKEN_ENV_VAR) + 1:].strip()
+    return ""
+
+
+def resolve_token(token: Optional[str] = None) -> str:
+    """Resolve a Diffbot API token from the explicit argument, env var, or file.
+
+    Returns an empty string if no token can be found.
+    """
+    if token and token.strip():
+        return token.strip()
+
+    env_token = os.environ.get(TOKEN_ENV_VAR, "").strip()
+    if env_token:
+        return env_token
+
+    return _read_credentials_file()

diffbot_python-0.2.0/src/diffbot/cli/_common.py

@@ -0,0 +1,21 @@
+import click
+
+from diffbot import Diffbot, resolve_token
+from diffbot._auth import CREDENTIALS_PATH, TOKEN_ENV_VAR
+
+
+def get_client() -> Diffbot:
+    """Build a Diffbot client using the shared credential resolution chain.
+
+    Looks at the DIFFBOT_API_TOKEN env var, then ~/.diffbot/credentials.
+    """
+    token = resolve_token()
+    if not token:
+        click.echo(
+            "Error: no Diffbot API token found.\n"
+            f"  Set a {TOKEN_ENV_VAR} environment variable, or\n"
+            f"  write '{TOKEN_ENV_VAR}=YOUR_TOKEN' to {CREDENTIALS_PATH}",
+            err=True,
+        )
+        raise click.Abort()
+    return Diffbot(token=token)

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/src/diffbot/cli/dql.py

@@ -15,7 +15,9 @@ from rich.table import Table
 from diffbot import DiffbotError
 
 from . import ontology
-from ._common import get_client, resolve_token
+from diffbot import resolve_token
+
+from ._common import get_client
 
 
 class _DqlGroup(click.Group):

diffbot_python-0.2.0/src/diffbot/cli/ontology.py

@@ -0,0 +1,74 @@
+"""CLI-side ontology access: a disk cache over the storage-agnostic core.
+
+The navigation logic lives in :mod:`diffbot.ontology` (the `Ontology` class).
+This module adds the CLI's caching policy on top: the ontology is read once from
+``~/.diffbot/ontology.json`` (populated by `db dql init`) and held in
+``_CACHE``. The module-level functions preserve the historical CLI surface and
+simply delegate to an `Ontology` built from the cached document.
+"""
+
+import json
+import pathlib
+from typing import Any, Dict, List, Optional, Tuple
+
+from diffbot.ontology import Ontology
+
+ONTOLOGY_PATH = pathlib.Path.home() / ".diffbot" / "ontology.json"
+
+_CACHE: Dict[str, Any] = {}
+
+
+def _data() -> Dict[str, Any]:
+    if "data" not in _CACHE:
+        if not ONTOLOGY_PATH.exists():
+            raise FileNotFoundError(
+                f"Ontology not found at {ONTOLOGY_PATH}. Run: db dql init"
+            )
+        _CACHE["data"] = json.loads(ONTOLOGY_PATH.read_text())
+    return _CACHE["data"]
+
+
+def _ontology() -> Ontology:
+    return Ontology(_data())
+
+
+def list_types() -> List[str]:
+    return _ontology().types()
+
+
+def list_composites() -> List[str]:
+    return _ontology().composites()
+
+
+def list_enums() -> List[str]:
+    return _ontology().enums()
+
+
+def list_taxonomies() -> List[str]:
+    return _ontology().taxonomies()
+
+
+def fields_for(type_name: str) -> Dict[str, Any]:
+    return _ontology().fields_for(type_name)
+
+
+def format_field(name: str, meta: Dict[str, Any]) -> str:
+    return Ontology.format_field(name, meta)
+
+
+def filter_fields(
+    fields: Dict[str, Any], search: Optional[str], include_deprecated: bool = False
+) -> List[Tuple[str, Dict[str, Any]]]:
+    return Ontology.filter_fields(fields, search, include_deprecated=include_deprecated)
+
+
+def taxonomy_values(name: str, search: Optional[str] = None) -> List[str]:
+    return _ontology().taxonomy_values(name, search)
+
+
+def enum_values(name: str) -> List[str]:
+    return _ontology().enum_values(name)
+
+
+def find_named(search: str) -> List[str]:
+    return _ontology().find_named(search)

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/src/diffbot/client.py

@@ -24,9 +24,13 @@ from .crawl import (
 from .kg import (
     dql as _dql,
     dql_async as _dql_async,
+    dql_fetch_ontology as _dql_fetch_ontology,
+    dql_fetch_ontology_async as _dql_fetch_ontology_async,
     dql_parallel as _dql_parallel,
+    dql_parallel_async as _dql_parallel_async,
     dql_refresh_ontology as _dql_refresh_ontology,
 )
+from .ontology import Ontology
 from .web_search import (
     WEB_SEARCH_BASE,
     web_search as _web_search,
@@ -48,8 +52,8 @@ class Diffbot:
     """Client for the Diffbot APIs.
 
     Example:
-        >>> from diffbot import Diffbot
-        >>> db = Diffbot(token=os.getenv("DIFFBOT_API_TOKEN"))
+        >>> from diffbot import Diffbot, resolve_token
+        >>> db = Diffbot(token=resolve_token())  # env var or ~/.diffbot/credentials
         >>> db.extract("https://example.com")
     """
 
@@ -155,6 +159,10 @@ class Diffbot:
         """Download the Diffbot Knowledge Graph ontology and write it to dest."""
         _dql_refresh_ontology(self, dest)
 
+    def dql_fetch_ontology(self) -> Ontology:
+        """Download the ontology and return it as a queryable Ontology (no caching)."""
+        return _dql_fetch_ontology(self)
+
     def web_search(self, text: str, *, num_results: Optional[int] = None, max_tokens: Optional[int] = None) -> Dict[str, Any]:
         """Search the web via the Diffbot LLM web search API."""
         return _web_search(self, text, num_results=num_results, max_tokens=max_tokens)
@@ -272,6 +280,14 @@ class DiffbotAsync:
         """
         return await _dql_async(self, query, size=size, from_=from_, format=format, filter=filter, exportspec=exportspec, extra=extra, raw=raw)
 
+    async def dql_parallel(self, queries: Sequence[Dict[str, Any]], *, workers: int = 8) -> List[Union[Dict[str, Any], bytes]]:
+        """Run multiple DQL queries concurrently. Each item is a dict of dql() keyword args."""
+        return await _dql_parallel_async(self, queries, workers=workers)
+
+    async def dql_fetch_ontology(self) -> Ontology:
+        """Download the ontology and return it as a queryable Ontology (no caching)."""
+        return await _dql_fetch_ontology_async(self)
+
     async def web_search(self, text: str, *, num_results: Optional[int] = None, max_tokens: Optional[int] = None) -> Dict[str, Any]:
         """Search the web via the Diffbot LLM web search API."""
         return await _web_search_async(self, text, num_results=num_results, max_tokens=max_tokens)

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/src/diffbot/kg.py

@@ -1,9 +1,12 @@
 """Diffbot Knowledge Graph APIs: DQL search and entity enhancement."""
 
+import asyncio
 import pathlib
 from concurrent.futures import ThreadPoolExecutor
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
 
+from .ontology import Ontology
+
 if TYPE_CHECKING:
     from .client import Diffbot, DiffbotAsync
 
@@ -83,8 +86,43 @@ def dql_parallel(
         return list(ex.map(lambda q: dql(client, **q), queries))
 
 
+async def dql_parallel_async(
+    client: "DiffbotAsync",
+    queries: Sequence[Dict[str, Any]],
+    *,
+    workers: int = 8,
+) -> List[Union[Dict[str, Any], bytes]]:
+    if not queries:
+        return []
+    sem = asyncio.Semaphore(workers)
+
+    async def _one(q: Dict[str, Any]) -> Union[Dict[str, Any], bytes]:
+        async with sem:
+            return await dql_async(client, **q)
+
+    return await asyncio.gather(*(_one(q) for q in queries))
+
+
 def dql_refresh_ontology(client: "Diffbot", dest: pathlib.Path) -> None:
     response = client._http.get(KG_ONTOLOGY_ENDPOINT)
     client._raise_for_status(response)
     dest.parent.mkdir(parents=True, exist_ok=True)
     dest.write_bytes(response.content)
+
+
+def dql_fetch_ontology(client: "Diffbot") -> Ontology:
+    """Download the ontology and return it as a queryable :class:`Ontology`.
+
+    Performs no caching — the caller decides whether and where to hold onto the
+    result. Use :func:`dql_refresh_ontology` instead to persist raw bytes to disk.
+    """
+    response = client._http.get(KG_ONTOLOGY_ENDPOINT)
+    client._raise_for_status(response)
+    return Ontology.from_json(response.content)
+
+
+async def dql_fetch_ontology_async(client: "DiffbotAsync") -> Ontology:
+    """Async variant of :func:`dql_fetch_ontology`."""
+    response = await client._http.get(KG_ONTOLOGY_ENDPOINT)
+    client._raise_for_status(response)
+    return Ontology.from_json(response.content)

diffbot_python-0.2.0/src/diffbot/ontology.py

@@ -0,0 +1,160 @@
+"""In-memory navigation of the Diffbot Knowledge Graph ontology.
+
+The ontology is a JSON document describing the Knowledge Graph's entity types,
+composite types, enums, and taxonomies. An agent constructing DQL needs it to
+look up real field paths and taxonomy values instead of guessing them.
+
+This module is pure and storage-agnostic: build an :class:`Ontology` from
+already-parsed data (or from raw JSON / a file path) and query it. How the
+ontology document is fetched, and whether or where it is cached, is left
+entirely to the caller — the `db` CLI caches it on disk at
+``~/.diffbot/ontology.json``; an in-process consumer (e.g. langchain) can cache
+the :class:`Ontology` in memory. Fetch a fresh one over HTTP with
+:meth:`diffbot.Diffbot.dql_fetch_ontology`.
+"""
+
+import json
+import pathlib
+import re
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+
+class Ontology:
+    """Queryable view over a parsed Diffbot ontology document.
+
+    The instance holds the parsed document on :attr:`data` and exposes pure
+    lookup methods over it. Nothing here performs I/O — construct with already
+    parsed data, or use :meth:`from_json` / :meth:`from_path` for convenience.
+    """
+
+    def __init__(self, data: Dict[str, Any]):
+        self.data = data
+
+    @classmethod
+    def from_json(cls, raw: Union[str, bytes]) -> "Ontology":
+        """Build from a raw JSON string or bytes (e.g. an HTTP response body)."""
+        return cls(json.loads(raw))
+
+    @classmethod
+    def from_path(cls, path: Union[str, pathlib.Path]) -> "Ontology":
+        """Build from a JSON file on disk."""
+        return cls(json.loads(pathlib.Path(path).read_text()))
+
+    def types(self) -> List[str]:
+        """All entity type names (e.g. ``Organization``, ``Person``)."""
+        return sorted(self.data.get("types", {}).keys())
+
+    def composites(self) -> List[str]:
+        """All composite type names (e.g. ``Location``, ``Employment``)."""
+        return sorted(self.data.get("composites", {}).keys())
+
+    def enums(self) -> List[str]:
+        """All enum type names (e.g. ``Language``, ``Gender``)."""
+        return sorted(self.data.get("enums", {}).keys())
+
+    def taxonomies(self) -> List[str]:
+        """All taxonomy names (e.g. ``OrganizationCategory``)."""
+        return sorted(self.data.get("taxonomies", {}).keys())
+
+    @staticmethod
+    def _fields_of(container: Dict[str, Any], type_name: str) -> Dict[str, Any]:
+        entry = container.get(type_name)
+        if entry is None:
+            raise KeyError(f"Unknown name: {type_name}")
+        return entry.get("fields", {})
+
+    def fields_for(self, type_name: str) -> Dict[str, Any]:
+        """Return the field map of an entity type or composite.
+
+        Auto-routes: ``type_name`` may be an entity type (``Organization``) or a
+        composite (``Location``). Raises ``KeyError`` if it is neither.
+        """
+        types = self.data.get("types", {})
+        composites = self.data.get("composites", {})
+        if type_name in types:
+            return self._fields_of(types, type_name)
+        if type_name in composites:
+            return self._fields_of(composites, type_name)
+        raise KeyError(f"{type_name} is not a known entity type or composite")
+
+    @staticmethod
+    def filter_fields(
+        fields: Dict[str, Any],
+        search: Optional[str],
+        include_deprecated: bool = False,
+    ) -> List[Tuple[str, Dict[str, Any]]]:
+        """Filter a field map by a name regex, dropping deprecated by default."""
+        pattern = re.compile(search, re.IGNORECASE) if search else None
+        out = []
+        for name, meta in fields.items():
+            if not include_deprecated and meta.get("isDeprecated"):
+                continue
+            if pattern and not pattern.search(name):
+                continue
+            out.append((name, meta))
+        return out
+
+    def taxonomy_values(self, name: str, search: Optional[str] = None) -> List[str]:
+        """Flatten a taxonomy's values (recursing into children), optionally filtered."""
+        tax = self.data.get("taxonomies", {}).get(name)
+        if tax is None:
+            raise KeyError(f"Unknown taxonomy: {name}")
+        pattern = re.compile(search, re.IGNORECASE) if search else None
+        out: List[str] = []
+
+        def walk(node: Dict[str, Any]) -> None:
+            n = node.get("name")
+            if n and (pattern is None or pattern.search(n)):
+                out.append(n)
+            for child in node.get("children", []) or []:
+                walk(child)
+
+        for cat in tax.get("categories", []) or []:
+            walk(cat)
+        return out
+
+    def enum_values(self, name: str) -> List[str]:
+        """Return the allowed values of an enum."""
+        enum = self.data.get("enums", {}).get(name)
+        if enum is None:
+            raise KeyError(f"Unknown enum: {name}")
+        return list(enum.get("values", []))
+
+    def find_named(self, search: str) -> List[str]:
+        """Fallback search: every ``name`` anywhere in the document matching a regex."""
+        pattern = re.compile(search, re.IGNORECASE)
+        found = set()
+
+        def walk(node: Any) -> None:
+            if isinstance(node, dict):
+                n = node.get("name")
+                if isinstance(n, str) and pattern.search(n):
+                    found.add(n)
+                for v in node.values():
+                    walk(v)
+            elif isinstance(node, list):
+                for v in node:
+                    walk(v)
+
+        walk(self.data)
+        return sorted(found)
+
+    @staticmethod
+    def format_field(name: str, meta: Dict[str, Any]) -> str:
+        """Render one field as ``<name>: [<type>] [flags...]`` for display."""
+        t = meta.get("type", "?")
+        if t == "LinkedEntity":
+            le = meta.get("leType") or []
+            if le:
+                t = f"LinkedEntity ({le[0]})"
+        flags = []
+        if meta.get("isList"):
+            flags.append("isList")
+        if meta.get("isComposite"):
+            flags.append("isComposite")
+        if meta.get("isEnum"):
+            flags.append("isEnum")
+        if meta.get("isDeprecated"):
+            flags.append("DEPRECATED")
+        suffix = "".join(f" [{f}]" for f in flags)
+        return f"{name}: [{t}]{suffix}"

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/src/diffbot/web_search.py

@@ -18,7 +18,7 @@ def web_search(
     headers = {"Authorization": f"Bearer {client.token}"}
     params: Dict[str, Any] = {"text": text}
     if num_results is not None:
-        params["num_results"] = num_results
+        params["size"] = num_results
     if max_tokens is not None:
         params["maxTokens"] = max_tokens
     response = client._http.get(client.web_search_url, headers=headers, params=params)
@@ -36,7 +36,7 @@ async def web_search_async(
     headers = {"Authorization": f"Bearer {client.token}"}
     params: Dict[str, Any] = {"text": text}
     if num_results is not None:
-        params["num_results"] = num_results
+        params["size"] = num_results
     if max_tokens is not None:
         params["maxTokens"] = max_tokens
     response = await client._http.get(client.web_search_url, headers=headers, params=params)

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/tests/conftest.py

@@ -1,15 +1,13 @@
-import os
-
 import pytest
 
-from diffbot import Diffbot
+from diffbot import Diffbot, resolve_token
 
 
 @pytest.fixture(scope="session")
 def live_token():
-    token = os.environ.get("DIFFBOT_TOKEN")
+    token = resolve_token()
     if not token:
-        pytest.skip("DIFFBOT_TOKEN not set")
+        pytest.skip("no Diffbot token found (set DIFFBOT_API_TOKEN or ~/.diffbot/credentials)")
     return token
 
 

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/tests/test_extract.py

@@ -1,7 +1,15 @@
 import httpx
 import pytest
 
-from diffbot import APIError, AuthError, Diffbot, ExtractionError, RateLimitError, ValidationError
+from diffbot import (
+    APIError,
+    AuthError,
+    Diffbot,
+    ExtractionError,
+    RateLimitError,
+    ValidationError,
+    resolve_token,
+)
 
 
 """
@@ -40,6 +48,36 @@ def test_token_required():
         Diffbot(token="")
 
 
+def test_resolve_token_explicit_wins(monkeypatch, tmp_path):
+    # An explicit token takes precedence over env var and file.
+    monkeypatch.setenv("DIFFBOT_API_TOKEN", "env-token")
+    assert resolve_token("explicit-token") == "explicit-token"
+
+
+def test_resolve_token_from_env(monkeypatch, tmp_path):
+    # A token in the environment is returned when none is passed.
+    monkeypatch.setenv("DIFFBOT_API_TOKEN", "env-token")
+    monkeypatch.setattr("diffbot._auth.CREDENTIALS_PATH", tmp_path / "missing")
+    assert resolve_token() == "env-token"
+    # And can be used to build a client.
+    assert Diffbot(token=resolve_token()).token == "env-token"
+
+
+def test_resolve_token_from_credentials_file(monkeypatch, tmp_path):
+    # Falls back to ~/.diffbot/credentials when no env var is set.
+    monkeypatch.delenv("DIFFBOT_API_TOKEN", raising=False)
+    creds = tmp_path / "credentials"
+    creds.write_text("DIFFBOT_API_TOKEN=file-token\n")
+    monkeypatch.setattr("diffbot._auth.CREDENTIALS_PATH", creds)
+    assert resolve_token() == "file-token"
+
+
+def test_resolve_token_missing_returns_empty(monkeypatch, tmp_path):
+    monkeypatch.delenv("DIFFBOT_API_TOKEN", raising=False)
+    monkeypatch.setattr("diffbot._auth.CREDENTIALS_PATH", tmp_path / "missing")
+    assert resolve_token() == ""
+
+
 def test_user_agent_header():
     captured = {}
 

diffbot_python-0.2.0/tests/test_ontology.py

@@ -0,0 +1,119 @@
+import json
+
+import httpx
+import pytest
+
+from diffbot import Diffbot, DiffbotAsync, Ontology
+
+FIXTURE_ONTOLOGY = {
+    "types": {
+        "Organization": {
+            "fields": {
+                "name": {"type": "String"},
+                "location": {"type": "Location", "isComposite": True},
+                "oldField": {"type": "String", "isDeprecated": True},
+            }
+        },
+        "Person": {"fields": {"name": {"type": "String"}}},
+    },
+    "composites": {
+        "Location": {"fields": {"city": {"type": "City", "isComposite": True}}},
+    },
+    "enums": {"Language": {"values": ["EN", "FR", "DE"]}},
+    "taxonomies": {
+        "OrganizationCategory": {
+            "categories": [
+                {"name": "Technology", "children": [{"name": "Semiconductor Companies"}]},
+            ]
+        }
+    },
+}
+
+
+@pytest.fixture
+def ont() -> Ontology:
+    return Ontology(FIXTURE_ONTOLOGY)
+
+
+def test_navigation_helpers(ont):
+    assert ont.types() == ["Organization", "Person"]
+    assert ont.composites() == ["Location"]
+    assert ont.enums() == ["Language"]
+    assert ont.taxonomies() == ["OrganizationCategory"]
+    assert ont.enum_values("Language") == ["EN", "FR", "DE"]
+    assert ont.taxonomy_values("OrganizationCategory", "semi") == ["Semiconductor Companies"]
+    assert ont.find_named("compan") == ["Semiconductor Companies"]
+
+
+def test_fields_for_routes_types_and_composites(ont):
+    assert "name" in ont.fields_for("Organization")
+    assert "city" in ont.fields_for("Location")
+    with pytest.raises(KeyError):
+        ont.fields_for("NopeType")
+
+
+def test_filter_fields_drops_deprecated_by_default(ont):
+    fields = ont.fields_for("Organization")
+    names = [n for n, _ in Ontology.filter_fields(fields, None)]
+    assert "oldField" not in names
+    names_incl = [n for n, _ in Ontology.filter_fields(fields, None, include_deprecated=True)]
+    assert "oldField" in names_incl
+
+
+def test_format_field(ont):
+    fields = ont.fields_for("Organization")
+    assert Ontology.format_field("location", fields["location"]) == "location: [Location] [isComposite]"
+
+
+def test_from_json_and_from_path(tmp_path):
+    raw = json.dumps(FIXTURE_ONTOLOGY)
+    assert Ontology.from_json(raw).types() == ["Organization", "Person"]
+    path = tmp_path / "ontology.json"
+    path.write_text(raw)
+    assert Ontology.from_path(path).enums() == ["Language"]
+
+
+def test_unknown_taxonomy_and_enum_raise(ont):
+    with pytest.raises(KeyError):
+        ont.taxonomy_values("Nope")
+    with pytest.raises(KeyError):
+        ont.enum_values("Nope")
+
+
+def test_dql_fetch_ontology_returns_ontology():
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.path.endswith("/ontology")
+        return httpx.Response(200, json=FIXTURE_ONTOLOGY)
+
+    db = Diffbot(token="test-token", transport=httpx.MockTransport(handler))
+    ont = db.dql_fetch_ontology()
+    assert isinstance(ont, Ontology)
+    assert ont.types() == ["Organization", "Person"]
+
+
+@pytest.mark.anyio
+async def test_async_dql_fetch_ontology_returns_ontology():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, json=FIXTURE_ONTOLOGY)
+
+    db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler))
+    ont = await db.dql_fetch_ontology()
+    assert isinstance(ont, Ontology)
+    assert ont.composites() == ["Location"]
+
+
+@pytest.mark.anyio
+async def test_async_dql_parallel_runs_all_queries():
+    def handler(request: httpx.Request) -> httpx.Response:
+        q = request.url.params["query"]
+        hits = 5 if "Diffbot" in q else 100
+        return httpx.Response(200, json={"hits": hits, "results": 0})
+
+    db = DiffbotAsync(token="test-token", transport=httpx.MockTransport(handler))
+    results = await db.dql_parallel(
+        [
+            {"query": 'type:Organization name:"Diffbot"', "size": 0},
+            {"query": "type:Organization", "size": 0},
+        ]
+    )
+    assert [r["hits"] for r in results] == [5, 100]

{diffbot_python-0.1.0 → diffbot_python-0.2.0}/tests/test_readme_examples.py

@@ -2,7 +2,7 @@
 import httpx
 import pytest
 
-from diffbot import CrawlEventType, Diffbot, DiffbotAsync
+from diffbot import CrawlEventType, Diffbot, DiffbotAsync, resolve_token
 
 SSE_PARIS = 'data: {"choices": [{"delta": {"content": "Paris"}}]}\n'
 
@@ -173,3 +173,21 @@ async def test_readme_async_entities():
     assert len(result["entities"]) == 2
     assert result["entities"][0]["name"] == "Apple"
     assert result["sentiment"] == 0.3
+
+
+# ---------------------------------------------------------------------------
+# Authentication
+# ---------------------------------------------------------------------------
+
+def test_readme_authentication_resolve_token(monkeypatch, tmp_path):
+    # README "Authentication": Diffbot(token=resolve_token()) using the env var.
+    monkeypatch.setenv("DIFFBOT_API_TOKEN", "test-token")
+    monkeypatch.setattr("diffbot._auth.CREDENTIALS_PATH", tmp_path / "missing")
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        assert request.url.params["token"] == "test-token"
+        return httpx.Response(200, json={"objects": [{"title": "Example"}]})
+
+    db = Diffbot(token=resolve_token(), transport=httpx.MockTransport(handler))
+    data = db.extract("https://www.example.com")
+    assert "objects" in data

diffbot_python-0.1.0/AGENTS.md

@@ -1,5 +0,0 @@
-# Agent Guidelines
-
-## README Examples
-
-Whenever a code example in `README.md` is added or updated, the corresponding test must be added or updated in `tests/test_readme_examples.py`. Run `python -m pytest tests/test_readme_examples.py` to validate before considering the work complete.

diffbot_python-0.1.0/CLAUDE.md

@@ -1 +0,0 @@
-AGENTS.md

diffbot_python-0.1.0/src/diffbot/cli/_common.py

@@ -1,36 +0,0 @@
-import os
-import pathlib
-
-import click
-
-from diffbot import Diffbot
-
-CREDENTIALS_PATH = pathlib.Path.home() / ".diffbot" / "credentials"
-
-
-def resolve_token() -> str:
-    """Return the Diffbot API token from the env var, falling back to ~/.diffbot/credentials."""
-    token = os.environ.get("DIFFBOT_API_TOKEN", "").strip()
-    if token:
-        return token
-
-    if CREDENTIALS_PATH.exists():
-        for line in CREDENTIALS_PATH.read_text().splitlines():
-            line = line.strip()
-            if line.startswith("DIFFBOT_API_TOKEN="):
-                return line[len("DIFFBOT_API_TOKEN="):].strip()
-
-    return ""
-
-
-def get_client() -> Diffbot:
-    token = resolve_token()
-    if not token:
-        click.echo(
-            "Error: no Diffbot API token found.\n"
-            "  Set a DIFFBOT_API_TOKEN environment variable, or\n"
-            f"  write 'DIFFBOT_API_TOKEN=YOUR_TOKEN' to {CREDENTIALS_PATH}",
-            err=True,
-        )
-        raise click.Abort()
-    return Diffbot(token=token)

diffbot_python-0.1.0/src/diffbot/cli/ontology.py

@@ -1,130 +0,0 @@
-import json
-import pathlib
-import re
-from typing import Any, Dict, List, Optional
-
-ONTOLOGY_PATH = pathlib.Path.home() / ".diffbot" / "ontology.json"
-
-_CACHE: Dict[str, Any] = {}
-
-
-def load() -> Dict[str, Any]:
-    if "data" not in _CACHE:
-        if not ONTOLOGY_PATH.exists():
-            raise FileNotFoundError(
-                f"Ontology not found at {ONTOLOGY_PATH}. Run: db dql init"
-            )
-        _CACHE["data"] = json.loads(ONTOLOGY_PATH.read_text())
-    return _CACHE["data"]
-
-
-def list_types() -> List[str]:
-    return sorted(load().get("types", {}).keys())
-
-
-def list_composites() -> List[str]:
-    return sorted(load().get("composites", {}).keys())
-
-
-def list_enums() -> List[str]:
-    return sorted(load().get("enums", {}).keys())
-
-
-def list_taxonomies() -> List[str]:
-    return sorted(load().get("taxonomies", {}).keys())
-
-
-def _fields_of(container: Dict[str, Any], type_name: str) -> Dict[str, Any]:
-    entry = container.get(type_name)
-    if entry is None:
-        raise KeyError(f"Unknown name: {type_name}")
-    return entry.get("fields", {})
-
-
-def fields_for(type_name: str) -> Dict[str, Any]:
-    data = load()
-    types = data.get("types", {})
-    composites = data.get("composites", {})
-    if type_name in types:
-        return _fields_of(types, type_name)
-    if type_name in composites:
-        return _fields_of(composites, type_name)
-    raise KeyError(f"{type_name} is not a known entity type or composite")
-
-
-def format_field(name: str, meta: Dict[str, Any]) -> str:
-    t = meta.get("type", "?")
-    if t == "LinkedEntity":
-        le = meta.get("leType") or []
-        if le:
-            t = f"LinkedEntity ({le[0]})"
-    flags = []
-    if meta.get("isList"):
-        flags.append("isList")
-    if meta.get("isComposite"):
-        flags.append("isComposite")
-    if meta.get("isEnum"):
-        flags.append("isEnum")
-    if meta.get("isDeprecated"):
-        flags.append("DEPRECATED")
-    suffix = "".join(f" [{f}]" for f in flags)
-    return f"{name}: [{t}]{suffix}"
-
-
-def filter_fields(fields: Dict[str, Any], search: Optional[str], include_deprecated: bool = False) -> List[tuple]:
-    pattern = re.compile(search, re.IGNORECASE) if search else None
-    out = []
-    for name, meta in fields.items():
-        if not include_deprecated and meta.get("isDeprecated"):
-            continue
-        if pattern and not pattern.search(name):
-            continue
-        out.append((name, meta))
-    return out
-
-
-def taxonomy_values(name: str, search: Optional[str] = None) -> List[str]:
-    data = load()
-    tax = data.get("taxonomies", {}).get(name)
-    if tax is None:
-        raise KeyError(f"Unknown taxonomy: {name}")
-    pattern = re.compile(search, re.IGNORECASE) if search else None
-    out: List[str] = []
-
-    def walk(node: Dict[str, Any]) -> None:
-        n = node.get("name")
-        if n and (pattern is None or pattern.search(n)):
-            out.append(n)
-        for child in node.get("children", []) or []:
-            walk(child)
-
-    for cat in tax.get("categories", []) or []:
-        walk(cat)
-    return out
-
-
-def enum_values(name: str) -> List[str]:
-    data = load()
-    enum = data.get("enums", {}).get(name)
-    if enum is None:
-        raise KeyError(f"Unknown enum: {name}")
-    return list(enum.get("values", []))
-
-
-def find_named(search: str) -> List[str]:
-    pattern = re.compile(search, re.IGNORECASE)
-    found = set()
-
-    def walk(node: Any) -> None:
-        if isinstance(node, dict):
-            n = node.get("name")
-            if isinstance(n, str) and pattern.search(n):
-                found.add(n)
-            for v in node.values():
-                walk(v)
-        elif isinstance(node, list):
-            for v in node:
-                walk(v)
-
-    walk(load())
-    return sorted(found)