datahub-describer 0.1.0__tar.gz

datahub_describer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 DataHub Describer contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

datahub_describer-0.1.0/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.4
+Name: datahub-describer
+Version: 0.1.0
+Summary: Hierarchical LLM-generated descriptions for DataHub
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: acryl-datahub[datahub-rest]>=1.7.0
+Requires-Dist: openai>=1.40.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# DataHub Describer
+
+Generate missing DataHub descriptions with an OpenAI-compatible LLM.
+
+The connector works from fields upward:
+
+```text
+fields → datasets → parent containers
+```
+
+Field prompts use field metadata. Parent prompts use the names and descriptions
+of their immediate children. Existing descriptions are preserved by default.
+
+## Install
+
+```bash
+pip install datahub-describer
+```
+
+## Configure
+
+Use the registered DataHub source type:
+
+```yaml
+source:
+  type: datahub-describer
+  config:
+    llm_model: "${LLM_MODEL}"
+    llm_base_url: "${LLM_BASE_URL}"
+    llm_api_key: "${LLM_API_KEY}"
+
+sink:
+  type: datahub-rest
+  config:
+    server: "${DATAHUB_GMS_URL}"
+```
+
+See `datahub_describer.example.yml` for all settings.
+
+## Development
+
+```bash
+pip install -e '.[dev]'
+pytest
+```

datahub_describer-0.1.0/README.md

@@ -0,0 +1,45 @@
+# DataHub Describer
+
+Generate missing DataHub descriptions with an OpenAI-compatible LLM.
+
+The connector works from fields upward:
+
+```text
+fields → datasets → parent containers
+```
+
+Field prompts use field metadata. Parent prompts use the names and descriptions
+of their immediate children. Existing descriptions are preserved by default.
+
+## Install
+
+```bash
+pip install datahub-describer
+```
+
+## Configure
+
+Use the registered DataHub source type:
+
+```yaml
+source:
+  type: datahub-describer
+  config:
+    llm_model: "${LLM_MODEL}"
+    llm_base_url: "${LLM_BASE_URL}"
+    llm_api_key: "${LLM_API_KEY}"
+
+sink:
+  type: datahub-rest
+  config:
+    server: "${DATAHUB_GMS_URL}"
+```
+
+See `datahub_describer.example.yml` for all settings.
+
+## Development
+
+```bash
+pip install -e '.[dev]'
+pytest
+```

datahub_describer-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "datahub-describer"
+version = "0.1.0"
+description = "Hierarchical LLM-generated descriptions for DataHub"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+  "acryl-datahub[datahub-rest]>=1.7.0",
+  "openai>=1.40.0",
+]
+
+[project.entry-points."datahub.ingestion.source.plugins"]
+datahub-describer = "datahub_describer.source:HierarchicalDocumentationSource"
+
+[project.optional-dependencies]
+dev = ["pytest"]
+
+[tool.setuptools.packages.find]
+where = ["src"]

datahub_describer-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

datahub_describer-0.1.0/src/datahub_describer/__init__.py

@@ -0,0 +1,6 @@
+from datahub_describer.source import (
+    HierarchicalDocumentationSource,
+)
+
+__all__ = ["HierarchicalDocumentationSource"]
+__version__ = "0.1.0"

datahub_describer-0.1.0/src/datahub_describer/llm_client.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, Dict, Optional
+
+from openai import OpenAI
+
+logger = logging.getLogger(__name__)
+
+
+def _json_object(content: str) -> Dict[str, Any]:
+    parsed = json.loads(content)
+    if not isinstance(parsed, dict):
+        raise ValueError("LLM response must be a JSON object")
+    return parsed
+
+
+def get_llm_client(
+    api_key: str,
+    base_url: str,
+    *,
+    timeout: float,
+    max_retries: int,
+) -> OpenAI:
+    return OpenAI(
+        api_key=api_key or "not-required",
+        base_url=base_url,
+        timeout=timeout,
+        max_retries=max_retries,
+    )
+
+
+def chat_json(
+    client: OpenAI,
+    *,
+    system: str,
+    user: str,
+    model: str,
+    temperature: Optional[float] = 0.1,
+) -> Dict[str, Any]:
+    messages = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+    kwargs: Dict[str, Any] = {
+        "model": model,
+        "response_format": {"type": "json_object"},
+        "messages": messages,
+    }
+    if temperature is not None:
+        kwargs["temperature"] = temperature
+
+    while True:
+        try:
+            response = client.chat.completions.create(**kwargs)
+            break
+        except Exception as exc:
+            message = str(exc).lower()
+            if "temperature" in kwargs and "temperature" in message:
+                logger.warning(
+                    "Model %s rejected temperature=%s; retrying without it",
+                    model,
+                    kwargs["temperature"],
+                )
+                kwargs.pop("temperature")
+                continue
+            if "response_format" in kwargs and (
+                "response_format" in message
+                or "response format" in message
+                or "json_object" in message
+            ):
+                logger.warning(
+                    "Model %s rejected JSON response format; retrying without it",
+                    model,
+                )
+                kwargs.pop("response_format")
+                continue
+            raise
+
+    content = response.choices[0].message.content or "{}"
+    try:
+        return _json_object(content)
+    except json.JSONDecodeError:
+        match = re.search(r"\{.*\}", content, re.DOTALL)
+        if not match:
+            logger.warning(
+                "Failed to parse LLM JSON response: %s",
+                content[:500],
+            )
+            return {}
+        return _json_object(match.group(0))