dg-kit 0.1.0__tar.gz

dg_kit-0.1.0/PKG-INFO

@@ -0,0 +1,99 @@
+Metadata-Version: 2.3
+Name: dg_kit
+Version: 0.1.0
+Summary: Data Governance Kit provides programmatic access to data governance metadata with integrations for different Logical and Physical Modeling tools.
+Author: Chelidze Georgii
+Author-email: Chelidze Georgii <chelidze.georgii.d@gmail.com>
+License: MIT
+Requires-Dist: pyyaml>=6.0.3 ; extra == 'dbt'
+Requires-Dist: notion-client>=2.7.0 ; extra == 'notion'
+Requires-Python: >=3.10
+Provides-Extra: dbt
+Provides-Extra: notion
+Description-Content-Type: text/markdown
+
+## Data Governance Kit (dg_kit)
+
+Data Governance Kit helps you access Data Governance information programmatically.
+It provides core objects that model Physical Model, Logical Model, Business Information,
+and related governance metadata. Integrations let you pull this data from tools like
+dbt, Oracle Data Modeler, and Notion, with more connectors planned in upcoming releases.
+
+This toolkit is handy for building Data Governance CI gates, strengthening Data Ops
+practices, and keeping governance checks close to your delivery workflows.
+
+## Requirements
+- Python >= 3.10
+
+## Install
+```bash
+pip install -e .
+```
+
+Optional extras:
+```bash
+pip install -e ".[dbt]"
+pip install -e ".[notion]"
+```
+
+## Quick Start
+
+### Parse an Oracle Data Modeler project
+```python
+from dg_kit.integrations.odm.parser import ODMParser
+
+parser = ODMParser("path/to/model.dmd")
+bi = parser.parse_bi()
+lm = parser.parse_lm()
+
+print(lm.version, len(lm.entities))
+```
+
+### Parse a dbt project into a physical model
+```python
+from dg_kit.integrations.dbt.parser import DBTParser
+
+pm = DBTParser("path/to/dbt_project").parse_pm()
+print(pm.version, len(pm.tables))
+```
+
+### Validate with conventions
+```python
+from dg_kit.base.convention import Convention, ConventionValidator
+from dg_kit.base.enums import ConventionRuleSeverity
+
+convention = Convention("example")
+
+@convention.rule(
+    name="has-entities",
+    severity=ConventionRuleSeverity.ERROR,
+    description="Logical model must contain at least one entity",
+)
+def has_entities(lm, pm):
+    return set() if lm.entities else {("no entities")}
+
+issues = ConventionValidator(lm, pm, convention).validate()
+```
+
+### Sync to Notion data catalog
+```python
+from dg_kit.integrations.notion.api import NotionDataCatalog
+
+catalog = NotionDataCatalog(
+    notion_token="secret",
+    dc_table_id="data_source_id",
+)
+rows = catalog.pull()
+print(len(rows))
+```
+
+## Development
+Run tests:
+```bash
+pytest
+```
+
+Export requirements with uv:
+```bash
+uv export --extra dbt --extra notion --group test -o requirements.txt
+```

dg_kit-0.1.0/README.md

@@ -0,0 +1,85 @@
+## Data Governance Kit (dg_kit)
+
+Data Governance Kit helps you access Data Governance information programmatically.
+It provides core objects that model Physical Model, Logical Model, Business Information,
+and related governance metadata. Integrations let you pull this data from tools like
+dbt, Oracle Data Modeler, and Notion, with more connectors planned in upcoming releases.
+
+This toolkit is handy for building Data Governance CI gates, strengthening Data Ops
+practices, and keeping governance checks close to your delivery workflows.
+
+## Requirements
+- Python >= 3.10
+
+## Install
+```bash
+pip install -e .
+```
+
+Optional extras:
+```bash
+pip install -e ".[dbt]"
+pip install -e ".[notion]"
+```
+
+## Quick Start
+
+### Parse an Oracle Data Modeler project
+```python
+from dg_kit.integrations.odm.parser import ODMParser
+
+parser = ODMParser("path/to/model.dmd")
+bi = parser.parse_bi()
+lm = parser.parse_lm()
+
+print(lm.version, len(lm.entities))
+```
+
+### Parse a dbt project into a physical model
+```python
+from dg_kit.integrations.dbt.parser import DBTParser
+
+pm = DBTParser("path/to/dbt_project").parse_pm()
+print(pm.version, len(pm.tables))
+```
+
+### Validate with conventions
+```python
+from dg_kit.base.convention import Convention, ConventionValidator
+from dg_kit.base.enums import ConventionRuleSeverity
+
+convention = Convention("example")
+
+@convention.rule(
+    name="has-entities",
+    severity=ConventionRuleSeverity.ERROR,
+    description="Logical model must contain at least one entity",
+)
+def has_entities(lm, pm):
+    return set() if lm.entities else {("no entities")}
+
+issues = ConventionValidator(lm, pm, convention).validate()
+```
+
+### Sync to Notion data catalog
+```python
+from dg_kit.integrations.notion.api import NotionDataCatalog
+
+catalog = NotionDataCatalog(
+    notion_token="secret",
+    dc_table_id="data_source_id",
+)
+rows = catalog.pull()
+print(len(rows))
+```
+
+## Development
+Run tests:
+```bash
+pytest
+```
+
+Export requirements with uv:
+```bash
+uv export --extra dbt --extra notion --group test -o requirements.txt
+```

dg_kit-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["uv_build>=0.9.26,<0.10.0"]
+build-backend = "uv_build"
+
+[project]
+name = "dg_kit"
+version = "0.1.0"
+description = "Data Governance Kit provides programmatic access to data governance metadata with integrations for different Logical and Physical Modeling tools."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Chelidze Georgii", email = "chelidze.georgii.d@gmail.com" }]
+
+[project.optional-dependencies]
+notion = [
+    "notion-client>=2.7.0",
+]
+dbt = [
+    "pyyaml>=6.0.3",
+]
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.5.1",
+    "pyright>=1.1.408",
+    "pytest>=9.0.2",
+    "ruff>=0.15.0",
+]
+test = [
+    "pytest>=9.0.2",
+]

dg_kit-0.1.0/src/dg_kit/base/business_information.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+from typing import Dict
+
+
+from dg_kit.base.dataclasses.business_information import (
+    Team,
+    Contact,
+    Document,
+    Email,
+    Url,
+)
+
+
+class BusinessInformation:
+    def __init__(self, version: str):
+        self.version = version
+        self.teams: Dict[str, Team] = {}
+        self.contacts: Dict[str, Contact] = {}
+        self.documents: Dict[str, Document] = {}
+        self.emails: Dict[str, Email] = {}
+        self.urls: Dict[str, Url] = {}
+        self.all_units_by_id: Dict[str, Team | Contact | Document | Email | Url] = {}
+        self.all_units_by_natural_key: Dict[
+            str, Team | Contact | Document | Email | Url
+        ] = {}
+
+    def register_team(self, team: Team) -> None:
+        self.teams[team.id] = team
+        self.all_units_by_id[team.id] = team
+        self.all_units_by_natural_key[team.natural_key] = team
+
+    def register_contact(self, contact: Contact) -> None:
+        self.contacts[contact.id] = contact
+        self.all_units_by_id[contact.id] = contact
+        self.all_units_by_natural_key[contact.natural_key] = contact
+
+    def register_document(self, document: Document) -> None:
+        self.documents[document.id] = document
+        self.all_units_by_id[document.id] = document
+        self.all_units_by_natural_key[document.natural_key] = document
+
+    def register_email(self, email: Email) -> None:
+        self.emails[email.id] = email
+        self.all_units_by_id[email.id] = email
+        self.all_units_by_natural_key[email.natural_key] = email
+
+    def register_url(self, url: Url) -> None:
+        self.urls[url.id] = url
+        self.all_units_by_id[url.id] = url
+        self.all_units_by_natural_key[url.natural_key] = url
+
+
+class BusinessInformationDatabase:
+    def __init__(self):
+        self.business_information: Dict[str, BusinessInformation] = {}
+
+    def register_business_information(
+        self, business_information: BusinessInformation
+    ) -> None:
+        self.business_information[business_information.version] = business_information

dg_kit-0.1.0/src/dg_kit/base/convention.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import List
+
+from dg_kit.base.physical_model import PhysicalModel
+from dg_kit.base.logical_model import LogicalModel
+from dg_kit.base.enums import ConventionRuleSeverity
+from dg_kit.base.dataclasses.convention import (
+    ConventionRule,
+    ConventionRuleFn,
+    ConventionBreach,
+)
+
+
+class Convention:
+    def __init__(self, name: str):
+        self.name = name
+        self._rules: List[ConventionRule] = []
+
+    def rule(
+        self,
+        name: str,
+        severity: ConventionRuleSeverity,
+        description: str,
+    ):
+        def decorator(fn: ConventionRuleFn) -> ConventionRuleFn:
+            self._rules.append(ConventionRule(name, severity, description, fn))
+            return fn
+
+        return decorator
+
+    @property
+    def rules(self) -> List[ConventionRule]:
+        return list(self._rules)
+
+
+class ConventionValidator:
+    def __init__(self, lm: LogicalModel, pm: PhysicalModel, convention: Convention):
+        self.lm = lm
+        self.pm = pm
+        self.convention = convention
+
+    def validate(self) -> List[ConventionBreach]:
+        issues: List[ConventionBreach] = []
+
+        for rule in self.convention.rules:
+            res = rule.fn(self.lm, self.pm)
+            for issue in res:
+                issue = ConventionBreach(
+                    severity=rule.severity,
+                    message=issue.message,
+                    unit_id=issue.unit_id,
+                    unit_natural_key=issue.unit_natural_key,
+                )
+                issues.append(issue)
+
+        return issues

dg_kit-0.1.0/src/dg_kit/base/data_catalog.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+from abc import ABC
+
+
+class DataCatalog(ABC): ...

dg_kit-0.1.0/src/dg_kit/base/dataclasses/__init__.py

@@ -0,0 +1,7 @@
+import hashlib
+
+
+def id_generator(*parts: str, size: int = 16) -> str:
+    h = hashlib.blake2b(digest_size=size)
+    h.update("\x1f".join(parts).encode("utf-8"))
+    return h.hexdigest()

dg_kit-0.1.0/src/dg_kit/base/dataclasses/business_information.py

@@ -0,0 +1,77 @@
+"""
+Docstring for dg_kit.base.dataclasses.business_information
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional, Tuple
+
+from dg_kit.base.dataclasses import id_generator
+
+
+@dataclass(frozen=True, slots=True)
+class SlackChannelUrl:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    url: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Email:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    email_address: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Url:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    url: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Contact:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    emails: Tuple[Email, ...]
+    urls: Tuple[Url, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Team:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    contacts: Tuple[Contact, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Document:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    reference: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))

dg_kit-0.1.0/src/dg_kit/base/dataclasses/convention.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Protocol, Set
+
+from dg_kit.base.enums import ConventionRuleSeverity
+from dg_kit.base.logical_model import LogicalModel
+from dg_kit.base.physical_model import PhysicalModel
+
+
+@dataclass(frozen=True, slots=True)
+class ConventionBreach:
+    severity: ConventionRuleSeverity
+    message: str
+    unit_id: Optional[str] = None
+    unit_natural_key: Optional[str] = None
+
+
+class ConventionRuleFn(Protocol):
+    def __call__(
+        self, lm: LogicalModel, pm: PhysicalModel
+    ) -> Set[ConventionBreach]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class ConventionRule:
+    name: str
+    severity: ConventionRuleSeverity
+    description: str
+    fn: ConventionRuleFn

dg_kit-0.1.0/src/dg_kit/base/dataclasses/data_catalog.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+from datetime import datetime
+
+from dataclasses import dataclass
+
+from dg_kit.base.enums import DataUnitType
+from dg_kit.base.dataclasses.business_information import Document, Team
+
+
+@dataclass(frozen=True, slots=True)
+class DataCatalogRow:
+    data_unit_name: str
+    data_unit_type: DataUnitType
+    domain: str
+    data_unit_uuid: str
+    last_edited_time: Optional[datetime] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class EntityTypeDataUnitPageInfo:
+    data_unit_type: DataUnitType
+    description: str
+    linked_documents: Tuple[Document, ...]
+    responsible_parties: Tuple[Team, ...]
+    master_source_systems: Tuple[str, ...]
+    core_layer_mapping: Tuple[str, ...]
+    pk_attributes_page_ids: Tuple[str, ...]
+    attributes_page_ids: Tuple[str, ...]
+    relationes_page_ids: Tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class AttributeTypeDataUnitPageInfo:
+    parent_entity_page_id: str
+    data_unit_type: DataUnitType
+    description: str
+    data_type: str
+    sensitivity_type: str
+    linked_documents: Tuple[Document, ...]
+    responsible_parties: Tuple[Team, ...]
+    core_layer_mapping: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class RelationTypeDataUnitPageInfo:
+    data_unit_type: DataUnitType
+    description: str
+    linked_documents: Tuple[Document, ...]
+    responsible_parties: Tuple[Team, ...]
+    core_layer_mapping: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    source_entity_page_id: str
+    target_entity_page_id: str
+    optional_source: bool
+    optional_target: bool
+    source_cardinality: str
+    target_cardinality: str

dg_kit-0.1.0/src/dg_kit/base/dataclasses/logical_model.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional, Tuple
+
+from dg_kit.base.dataclasses import id_generator
+from dg_kit.base.dataclasses.business_information import Team, Document
+
+
+@dataclass(frozen=True, slots=True)
+class EntityIdentifier:
+    id: str = field(init=False)
+    natural_key: str
+    name: Optional[str]
+    is_pk: bool
+    entity_id: str
+    used_attributes_ids: Tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Relation:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    domain: Optional[str]
+    description: str
+    pm_map: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    responsible_parties: Tuple[Team, ...]
+    documents: Tuple[Document, ...]
+    source_entity_id: str
+    target_entity_id: str
+    optional_source: Optional[bool]
+    optional_target: Optional[bool]
+    source_cardinality: Optional[str]
+    target_cardinality: Optional[str]
+    created_by: Optional[str] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Attribute:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    domain: Optional[str]
+    description: str
+    sensitivity_type: str
+    data_type: str
+    pm_map: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    responsible_parties: Tuple[Team, ...]
+    documents: Tuple[Document, ...]
+    entity_id: str
+    created_by: Optional[str] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Entity:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    domain: Optional[str]
+    description: str
+    identifiers: Tuple[EntityIdentifier, ...]
+    attributes: Tuple[str, ...]
+    pm_map: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    responsible_parties: Tuple[Team, ...]
+    documents: Tuple[Document, ...]
+    created_by: Optional[str] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))

dg_kit-0.1.0/src/dg_kit/base/dataclasses/physical_model.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass, field
+from dg_kit.base.dataclasses import id_generator
+
+
+@dataclass(frozen=True, slots=True)
+class Layer:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    is_landing: bool
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Table:
+    id: str = field(init=False)
+    natural_key: str
+    layer_id: str
+    name: str
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Column:
+    id: str = field(init=False)
+    natural_key: str
+    layer_id: str
+    table_id: str
+    name: str
+    data_type: str
+    description: str = ""
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))

dg_kit-0.1.0/src/dg_kit/base/enums.py

@@ -0,0 +1,13 @@
+from enum import StrEnum
+
+
+class ConventionRuleSeverity(StrEnum):
+    INFO = "info"
+    WARN = "warning"
+    ERROR = "error"
+
+
+class DataUnitType(StrEnum):
+    ENTITY = "entity"
+    ATTRIBUTE = "attribute"
+    RELATION = "relation"