diffbot 3.0.0__py3-none-any.whl

diffbot/extract.py

@@ -0,0 +1,45 @@
+"""Diffbot Analyze API: extract structured content from a URL."""
+
+from typing import TYPE_CHECKING, Any, Dict
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+from .errors import ExtractionError
+
+
+def _build_params(client: Any, url: str, fmt: str) -> Dict[str, Any]:
+    params = {"token": client.token, "url": url, "timeout": 30000}
+    if fmt == "markdown":
+        params["mode"] = "llm"
+    return params
+
+
+def _parse_response(client: Any, data: Dict[str, Any]) -> Dict[str, Any]:
+    if "errorCode" in data:
+        raise ExtractionError(data["errorCode"], data.get("error", ""))
+    return data
+
+
+def _normalize_url(url: str) -> str:
+    return url if url.startswith("http") else f"https://{url}"
+
+
+def extract(client: "Diffbot", url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]:
+    url = _normalize_url(url)
+    response = client._http.get(
+        f"{client.analyze_url}/{api}",
+        params=_build_params(client, url, fmt),
+    )
+    client._raise_for_status(response)
+    return _parse_response(client, response.json())
+
+
+async def extract_async(client: "DiffbotAsync", url: str, api: str = "analyze", fmt: str = "markdown") -> Dict[str, Any]:
+    url = _normalize_url(url)
+    response = await client._http.get(
+        f"{client.analyze_url}/{api}",
+        params=_build_params(client, url, fmt),
+    )
+    client._raise_for_status(response)
+    return _parse_response(client, response.json())

diffbot/kg.py

@@ -0,0 +1,128 @@
+"""Diffbot Knowledge Graph APIs: DQL search and entity enhancement."""
+
+import asyncio
+import pathlib
+from concurrent.futures import ThreadPoolExecutor
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Union
+
+from .ontology import Ontology
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+KG_DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql"
+KG_ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology"
+
+
+def _build_dql_params(
+    client: Any,
+    query: str,
+    size: int,
+    from_: int,
+    format: str,
+    filter: Optional[str],
+    exportspec: Optional[str],
+    extra: Optional[Dict[str, str]],
+) -> Dict[str, Any]:
+    params: Dict[str, Any] = {"token": client.token, "query": query, "size": size}
+    if from_:
+        params["from"] = from_
+    if format != "json":
+        params["format"] = format
+    if filter is not None:
+        params["filter"] = filter
+    if exportspec is not None:
+        params["exportspec"] = exportspec
+    if extra:
+        params.update(extra)
+    return params
+
+
+def dql(
+    client: "Diffbot",
+    query: str,
+    *,
+    size: int = 10,
+    from_: int = 0,
+    format: str = "json",
+    filter: Optional[str] = None,
+    exportspec: Optional[str] = None,
+    extra: Optional[Dict[str, str]] = None,
+    raw: bool = False,
+) -> Union[Dict[str, Any], bytes]:
+    params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra)
+    response = client._http.get(client.dql_url, params=params)
+    client._raise_for_status(response)
+    return response.content if raw else response.json()
+
+
+async def dql_async(
+    client: "DiffbotAsync",
+    query: str,
+    *,
+    size: int = 10,
+    from_: int = 0,
+    format: str = "json",
+    filter: Optional[str] = None,
+    exportspec: Optional[str] = None,
+    extra: Optional[Dict[str, str]] = None,
+    raw: bool = False,
+) -> Union[Dict[str, Any], bytes]:
+    params = _build_dql_params(client, query, size, from_, format, filter, exportspec, extra)
+    response = await client._http.get(client.dql_url, params=params)
+    client._raise_for_status(response)
+    return response.content if raw else response.json()
+
+
+def dql_parallel(
+    client: "Diffbot",
+    queries: Sequence[Dict[str, Any]],
+    *,
+    workers: int = 8,
+) -> List[Union[Dict[str, Any], bytes]]:
+    if not queries:
+        return []
+    with ThreadPoolExecutor(max_workers=min(workers, len(queries))) as ex:
+        return list(ex.map(lambda q: dql(client, **q), queries))
+
+
+async def dql_parallel_async(
+    client: "DiffbotAsync",
+    queries: Sequence[Dict[str, Any]],
+    *,
+    workers: int = 8,
+) -> List[Union[Dict[str, Any], bytes]]:
+    if not queries:
+        return []
+    sem = asyncio.Semaphore(workers)
+
+    async def _one(q: Dict[str, Any]) -> Union[Dict[str, Any], bytes]:
+        async with sem:
+            return await dql_async(client, **q)
+
+    return await asyncio.gather(*(_one(q) for q in queries))
+
+
+def dql_refresh_ontology(client: "Diffbot", dest: pathlib.Path) -> None:
+    response = client._http.get(client.ontology_url)
+    client._raise_for_status(response)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    dest.write_bytes(response.content)
+
+
+def dql_fetch_ontology(client: "Diffbot") -> Ontology:
+    """Download the ontology and return it as a queryable :class:`Ontology`.
+
+    Performs no caching — the caller decides whether and where to hold onto the
+    result. Use :func:`dql_refresh_ontology` instead to persist raw bytes to disk.
+    """
+    response = client._http.get(client.ontology_url)
+    client._raise_for_status(response)
+    return Ontology.from_json(response.content)
+
+
+async def dql_fetch_ontology_async(client: "DiffbotAsync") -> Ontology:
+    """Async variant of :func:`dql_fetch_ontology`."""
+    response = await client._http.get(client.ontology_url)
+    client._raise_for_status(response)
+    return Ontology.from_json(response.content)

diffbot/nlp.py

@@ -0,0 +1,37 @@
+"""Diffbot NLP API: entity identification, resolution, and sentiment."""
+
+from typing import TYPE_CHECKING, Any, Dict
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+NLP_BASE = "https://nl.diffbot.com/v1/"
+NLP_FIELDS = "entities,sentiment"
+
+
+def entities(
+    client: "Diffbot",
+    text: str,
+    *,
+    lang: str = "auto",
+) -> Dict[str, Any]:
+    params = {"token": client.token, "fields": NLP_FIELDS}
+    payload = [{"lang": lang, "format": "plain text", "content": text}]
+    response = client._http.post(client.nlp_url, params=params, json=payload)
+    client._raise_for_status(response)
+    data = response.json()
+    return data[0] if isinstance(data, list) else data
+
+
+async def entities_async(
+    client: "DiffbotAsync",
+    text: str,
+    *,
+    lang: str = "auto",
+) -> Dict[str, Any]:
+    params = {"token": client.token, "fields": NLP_FIELDS}
+    payload = [{"lang": lang, "format": "plain text", "content": text}]
+    response = await client._http.post(client.nlp_url, params=params, json=payload)
+    client._raise_for_status(response)
+    data = response.json()
+    return data[0] if isinstance(data, list) else data

diffbot/ontology.py

@@ -0,0 +1,160 @@
+"""In-memory navigation of the Diffbot Knowledge Graph ontology.
+
+The ontology is a JSON document describing the Knowledge Graph's entity types,
+composite types, enums, and taxonomies. An agent constructing DQL needs it to
+look up real field paths and taxonomy values instead of guessing them.
+
+This module is pure and storage-agnostic: build an :class:`Ontology` from
+already-parsed data (or from raw JSON / a file path) and query it. How the
+ontology document is fetched, and whether or where it is cached, is left
+entirely to the caller — the `db` CLI caches it on disk at
+``~/.diffbot/ontology.json``; an in-process consumer (e.g. langchain) can cache
+the :class:`Ontology` in memory. Fetch a fresh one over HTTP with
+:meth:`diffbot.Diffbot.dql_fetch_ontology`.
+"""
+
+import json
+import pathlib
+import re
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+
+class Ontology:
+    """Queryable view over a parsed Diffbot ontology document.
+
+    The instance holds the parsed document on :attr:`data` and exposes pure
+    lookup methods over it. Nothing here performs I/O — construct with already
+    parsed data, or use :meth:`from_json` / :meth:`from_path` for convenience.
+    """
+
+    def __init__(self, data: Dict[str, Any]):
+        self.data = data
+
+    @classmethod
+    def from_json(cls, raw: Union[str, bytes]) -> "Ontology":
+        """Build from a raw JSON string or bytes (e.g. an HTTP response body)."""
+        return cls(json.loads(raw))
+
+    @classmethod
+    def from_path(cls, path: Union[str, pathlib.Path]) -> "Ontology":
+        """Build from a JSON file on disk."""
+        return cls(json.loads(pathlib.Path(path).read_text()))
+
+    def types(self) -> List[str]:
+        """All entity type names (e.g. ``Organization``, ``Person``)."""
+        return sorted(self.data.get("types", {}).keys())
+
+    def composites(self) -> List[str]:
+        """All composite type names (e.g. ``Location``, ``Employment``)."""
+        return sorted(self.data.get("composites", {}).keys())
+
+    def enums(self) -> List[str]:
+        """All enum type names (e.g. ``Language``, ``Gender``)."""
+        return sorted(self.data.get("enums", {}).keys())
+
+    def taxonomies(self) -> List[str]:
+        """All taxonomy names (e.g. ``OrganizationCategory``)."""
+        return sorted(self.data.get("taxonomies", {}).keys())
+
+    @staticmethod
+    def _fields_of(container: Dict[str, Any], type_name: str) -> Dict[str, Any]:
+        entry = container.get(type_name)
+        if entry is None:
+            raise KeyError(f"Unknown name: {type_name}")
+        return entry.get("fields", {})
+
+    def fields_for(self, type_name: str) -> Dict[str, Any]:
+        """Return the field map of an entity type or composite.
+
+        Auto-routes: ``type_name`` may be an entity type (``Organization``) or a
+        composite (``Location``). Raises ``KeyError`` if it is neither.
+        """
+        types = self.data.get("types", {})
+        composites = self.data.get("composites", {})
+        if type_name in types:
+            return self._fields_of(types, type_name)
+        if type_name in composites:
+            return self._fields_of(composites, type_name)
+        raise KeyError(f"{type_name} is not a known entity type or composite")
+
+    @staticmethod
+    def filter_fields(
+        fields: Dict[str, Any],
+        search: Optional[str],
+        include_deprecated: bool = False,
+    ) -> List[Tuple[str, Dict[str, Any]]]:
+        """Filter a field map by a name regex, dropping deprecated by default."""
+        pattern = re.compile(search, re.IGNORECASE) if search else None
+        out = []
+        for name, meta in fields.items():
+            if not include_deprecated and meta.get("isDeprecated"):
+                continue
+            if pattern and not pattern.search(name):
+                continue
+            out.append((name, meta))
+        return out
+
+    def taxonomy_values(self, name: str, search: Optional[str] = None) -> List[str]:
+        """Flatten a taxonomy's values (recursing into children), optionally filtered."""
+        tax = self.data.get("taxonomies", {}).get(name)
+        if tax is None:
+            raise KeyError(f"Unknown taxonomy: {name}")
+        pattern = re.compile(search, re.IGNORECASE) if search else None
+        out: List[str] = []
+
+        def walk(node: Dict[str, Any]) -> None:
+            n = node.get("name")
+            if n and (pattern is None or pattern.search(n)):
+                out.append(n)
+            for child in node.get("children", []) or []:
+                walk(child)
+
+        for cat in tax.get("categories", []) or []:
+            walk(cat)
+        return out
+
+    def enum_values(self, name: str) -> List[str]:
+        """Return the allowed values of an enum."""
+        enum = self.data.get("enums", {}).get(name)
+        if enum is None:
+            raise KeyError(f"Unknown enum: {name}")
+        return list(enum.get("values", []))
+
+    def find_named(self, search: str) -> List[str]:
+        """Fallback search: every ``name`` anywhere in the document matching a regex."""
+        pattern = re.compile(search, re.IGNORECASE)
+        found = set()
+
+        def walk(node: Any) -> None:
+            if isinstance(node, dict):
+                n = node.get("name")
+                if isinstance(n, str) and pattern.search(n):
+                    found.add(n)
+                for v in node.values():
+                    walk(v)
+            elif isinstance(node, list):
+                for v in node:
+                    walk(v)
+
+        walk(self.data)
+        return sorted(found)
+
+    @staticmethod
+    def format_field(name: str, meta: Dict[str, Any]) -> str:
+        """Render one field as ``<name>: [<type>] [flags...]`` for display."""
+        t = meta.get("type", "?")
+        if t == "LinkedEntity":
+            le = meta.get("leType") or []
+            if le:
+                t = f"LinkedEntity ({le[0]})"
+        flags = []
+        if meta.get("isList"):
+            flags.append("isList")
+        if meta.get("isComposite"):
+            flags.append("isComposite")
+        if meta.get("isEnum"):
+            flags.append("isEnum")
+        if meta.get("isDeprecated"):
+            flags.append("DEPRECATED")
+        suffix = "".join(f" [{f}]" for f in flags)
+        return f"{name}: [{t}]{suffix}"

diffbot/web_search.py

@@ -0,0 +1,44 @@
+"""Diffbot web search API."""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+if TYPE_CHECKING:
+    from .client import Diffbot, DiffbotAsync
+
+WEB_SEARCH_BASE = "https://llm.diffbot.com/api/v1/web_search"
+
+
+def web_search(
+    client: "Diffbot",
+    text: str,
+    *,
+    num_results: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+) -> Dict[str, Any]:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    params: Dict[str, Any] = {"text": text}
+    if num_results is not None:
+        params["size"] = num_results
+    if max_tokens is not None:
+        params["maxTokens"] = max_tokens
+    response = client._http.get(client.web_search_url, headers=headers, params=params)
+    client._raise_for_status(response)
+    return response.json()
+
+
+async def web_search_async(
+    client: "DiffbotAsync",
+    text: str,
+    *,
+    num_results: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+) -> Dict[str, Any]:
+    headers = {"Authorization": f"Bearer {client.token}"}
+    params: Dict[str, Any] = {"text": text}
+    if num_results is not None:
+        params["size"] = num_results
+    if max_tokens is not None:
+        params["maxTokens"] = max_tokens
+    response = await client._http.get(client.web_search_url, headers=headers, params=params)
+    client._raise_for_status(response)
+    return response.json()