dg-kit 0.1.0__py3-none-any.whl

dg_kit/base/business_information.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+from typing import Dict
+
+
+from dg_kit.base.dataclasses.business_information import (
+    Team,
+    Contact,
+    Document,
+    Email,
+    Url,
+)
+
+
+class BusinessInformation:
+    def __init__(self, version: str):
+        self.version = version
+        self.teams: Dict[str, Team] = {}
+        self.contacts: Dict[str, Contact] = {}
+        self.documents: Dict[str, Document] = {}
+        self.emails: Dict[str, Email] = {}
+        self.urls: Dict[str, Url] = {}
+        self.all_units_by_id: Dict[str, Team | Contact | Document | Email | Url] = {}
+        self.all_units_by_natural_key: Dict[
+            str, Team | Contact | Document | Email | Url
+        ] = {}
+
+    def register_team(self, team: Team) -> None:
+        self.teams[team.id] = team
+        self.all_units_by_id[team.id] = team
+        self.all_units_by_natural_key[team.natural_key] = team
+
+    def register_contact(self, contact: Contact) -> None:
+        self.contacts[contact.id] = contact
+        self.all_units_by_id[contact.id] = contact
+        self.all_units_by_natural_key[contact.natural_key] = contact
+
+    def register_document(self, document: Document) -> None:
+        self.documents[document.id] = document
+        self.all_units_by_id[document.id] = document
+        self.all_units_by_natural_key[document.natural_key] = document
+
+    def register_email(self, email: Email) -> None:
+        self.emails[email.id] = email
+        self.all_units_by_id[email.id] = email
+        self.all_units_by_natural_key[email.natural_key] = email
+
+    def register_url(self, url: Url) -> None:
+        self.urls[url.id] = url
+        self.all_units_by_id[url.id] = url
+        self.all_units_by_natural_key[url.natural_key] = url
+
+
+class BusinessInformationDatabase:
+    def __init__(self):
+        self.business_information: Dict[str, BusinessInformation] = {}
+
+    def register_business_information(
+        self, business_information: BusinessInformation
+    ) -> None:
+        self.business_information[business_information.version] = business_information

dg_kit/base/convention.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import List
+
+from dg_kit.base.physical_model import PhysicalModel
+from dg_kit.base.logical_model import LogicalModel
+from dg_kit.base.enums import ConventionRuleSeverity
+from dg_kit.base.dataclasses.convention import (
+    ConventionRule,
+    ConventionRuleFn,
+    ConventionBreach,
+)
+
+
+class Convention:
+    def __init__(self, name: str):
+        self.name = name
+        self._rules: List[ConventionRule] = []
+
+    def rule(
+        self,
+        name: str,
+        severity: ConventionRuleSeverity,
+        description: str,
+    ):
+        def decorator(fn: ConventionRuleFn) -> ConventionRuleFn:
+            self._rules.append(ConventionRule(name, severity, description, fn))
+            return fn
+
+        return decorator
+
+    @property
+    def rules(self) -> List[ConventionRule]:
+        return list(self._rules)
+
+
+class ConventionValidator:
+    def __init__(self, lm: LogicalModel, pm: PhysicalModel, convention: Convention):
+        self.lm = lm
+        self.pm = pm
+        self.convention = convention
+
+    def validate(self) -> List[ConventionBreach]:
+        issues: List[ConventionBreach] = []
+
+        for rule in self.convention.rules:
+            res = rule.fn(self.lm, self.pm)
+            for issue in res:
+                issue = ConventionBreach(
+                    severity=rule.severity,
+                    message=issue.message,
+                    unit_id=issue.unit_id,
+                    unit_natural_key=issue.unit_natural_key,
+                )
+                issues.append(issue)
+
+        return issues

dg_kit/base/data_catalog.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+
+from abc import ABC
+
+
+class DataCatalog(ABC): ...

dg_kit/base/dataclasses/__init__.py

@@ -0,0 +1,7 @@
+import hashlib
+
+
+def id_generator(*parts: str, size: int = 16) -> str:
+    h = hashlib.blake2b(digest_size=size)
+    h.update("\x1f".join(parts).encode("utf-8"))
+    return h.hexdigest()

dg_kit/base/dataclasses/business_information.py

@@ -0,0 +1,77 @@
+"""
+Docstring for dg_kit.base.dataclasses.business_information
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Optional, Tuple
+
+from dg_kit.base.dataclasses import id_generator
+
+
+@dataclass(frozen=True, slots=True)
+class SlackChannelUrl:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    url: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Email:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    email_address: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Url:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    url: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Contact:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    emails: Tuple[Email, ...]
+    urls: Tuple[Url, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Team:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    contacts: Tuple[Contact, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Document:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    reference: Optional[str]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))

dg_kit/base/dataclasses/convention.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Protocol, Set
+
+from dg_kit.base.enums import ConventionRuleSeverity
+from dg_kit.base.logical_model import LogicalModel
+from dg_kit.base.physical_model import PhysicalModel
+
+
+@dataclass(frozen=True, slots=True)
+class ConventionBreach:
+    severity: ConventionRuleSeverity
+    message: str
+    unit_id: Optional[str] = None
+    unit_natural_key: Optional[str] = None
+
+
+class ConventionRuleFn(Protocol):
+    def __call__(
+        self, lm: LogicalModel, pm: PhysicalModel
+    ) -> Set[ConventionBreach]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class ConventionRule:
+    name: str
+    severity: ConventionRuleSeverity
+    description: str
+    fn: ConventionRuleFn

dg_kit/base/dataclasses/data_catalog.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import Optional, Tuple
+
+from datetime import datetime
+
+from dataclasses import dataclass
+
+from dg_kit.base.enums import DataUnitType
+from dg_kit.base.dataclasses.business_information import Document, Team
+
+
+@dataclass(frozen=True, slots=True)
+class DataCatalogRow:
+    data_unit_name: str
+    data_unit_type: DataUnitType
+    domain: str
+    data_unit_uuid: str
+    last_edited_time: Optional[datetime] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class EntityTypeDataUnitPageInfo:
+    data_unit_type: DataUnitType
+    description: str
+    linked_documents: Tuple[Document, ...]
+    responsible_parties: Tuple[Team, ...]
+    master_source_systems: Tuple[str, ...]
+    core_layer_mapping: Tuple[str, ...]
+    pk_attributes_page_ids: Tuple[str, ...]
+    attributes_page_ids: Tuple[str, ...]
+    relationes_page_ids: Tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class AttributeTypeDataUnitPageInfo:
+    parent_entity_page_id: str
+    data_unit_type: DataUnitType
+    description: str
+    data_type: str
+    sensitivity_type: str
+    linked_documents: Tuple[Document, ...]
+    responsible_parties: Tuple[Team, ...]
+    core_layer_mapping: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class RelationTypeDataUnitPageInfo:
+    data_unit_type: DataUnitType
+    description: str
+    linked_documents: Tuple[Document, ...]
+    responsible_parties: Tuple[Team, ...]
+    core_layer_mapping: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    source_entity_page_id: str
+    target_entity_page_id: str
+    optional_source: bool
+    optional_target: bool
+    source_cardinality: str
+    target_cardinality: str

dg_kit/base/dataclasses/logical_model.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional, Tuple
+
+from dg_kit.base.dataclasses import id_generator
+from dg_kit.base.dataclasses.business_information import Team, Document
+
+
+@dataclass(frozen=True, slots=True)
+class EntityIdentifier:
+    id: str = field(init=False)
+    natural_key: str
+    name: Optional[str]
+    is_pk: bool
+    entity_id: str
+    used_attributes_ids: Tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Relation:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    domain: Optional[str]
+    description: str
+    pm_map: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    responsible_parties: Tuple[Team, ...]
+    documents: Tuple[Document, ...]
+    source_entity_id: str
+    target_entity_id: str
+    optional_source: Optional[bool]
+    optional_target: Optional[bool]
+    source_cardinality: Optional[str]
+    target_cardinality: Optional[str]
+    created_by: Optional[str] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Attribute:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    domain: Optional[str]
+    description: str
+    sensitivity_type: str
+    data_type: str
+    pm_map: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    responsible_parties: Tuple[Team, ...]
+    documents: Tuple[Document, ...]
+    entity_id: str
+    created_by: Optional[str] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Entity:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    domain: Optional[str]
+    description: str
+    identifiers: Tuple[EntityIdentifier, ...]
+    attributes: Tuple[str, ...]
+    pm_map: Tuple[str, ...]
+    master_source_systems: Tuple[str, ...]
+    responsible_parties: Tuple[Team, ...]
+    documents: Tuple[Document, ...]
+    created_by: Optional[str] = None
+    created_time: Optional[datetime] = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))

dg_kit/base/dataclasses/physical_model.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass, field
+from dg_kit.base.dataclasses import id_generator
+
+
+@dataclass(frozen=True, slots=True)
+class Layer:
+    id: str = field(init=False)
+    natural_key: str
+    name: str
+    is_landing: bool
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Table:
+    id: str = field(init=False)
+    natural_key: str
+    layer_id: str
+    name: str
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))
+
+
+@dataclass(frozen=True, slots=True)
+class Column:
+    id: str = field(init=False)
+    natural_key: str
+    layer_id: str
+    table_id: str
+    name: str
+    data_type: str
+    description: str = ""
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "id", id_generator(self.natural_key))

dg_kit/base/enums.py

@@ -0,0 +1,13 @@
+from enum import StrEnum
+
+
+class ConventionRuleSeverity(StrEnum):
+    INFO = "info"
+    WARN = "warning"
+    ERROR = "error"
+
+
+class DataUnitType(StrEnum):
+    ENTITY = "entity"
+    ATTRIBUTE = "attribute"
+    RELATION = "relation"

dg_kit/base/logical_model.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+from typing import Dict, List
+
+
+from dg_kit.base.dataclasses.logical_model import Entity, Attribute, Relation
+
+
+class LogicalModel:
+    def __init__(self, version: str):
+        self.version = version
+        self.entities: Dict[str, Entity] = {}
+        self.attributes: Dict[str, Attribute] = {}
+        self.dependencies: dict[str, set[str]] = {}
+        self.relations: Dict[str, Relation] = {}
+        self.relations_ids_by_entity_id: Dict[str, List[Relation]] = {}
+        self.all_units_by_id: Dict[str, Entity | Attribute | Relation] = {}
+        self.all_units_by_natural_key: Dict[str, Entity | Attribute | Relation] = {}
+        self.pm_objects_nks_used: set[str] = set()
+        self.all_lm_units_by_odm_id: Dict[str, Entity | Attribute | Relation] = {}
+
+    def register_entity(self, entity: Entity) -> None:
+        self.entities[entity.id] = entity
+        self.all_units_by_id[entity.id] = entity
+        self.all_units_by_natural_key[entity.natural_key] = entity
+
+        for nk in entity.pm_map:
+            self.pm_objects_nks_used.add(nk)
+
+    def register_attribute(self, attribute: Attribute) -> None:
+        self.attributes[attribute.id] = attribute
+        self.all_units_by_id[attribute.id] = attribute
+        self.all_units_by_natural_key[attribute.natural_key] = attribute
+
+        for nk in attribute.pm_map:
+            self.pm_objects_nks_used.add(nk)
+
+    def register_relation(self, relation: Relation) -> None:
+        self.relations[relation.id] = relation
+        self.all_units_by_id[relation.id] = relation
+        self.all_units_by_natural_key[relation.natural_key] = relation
+        if relation.source_entity_id in self.relations_ids_by_entity_id:
+            self.relations_ids_by_entity_id[relation.source_entity_id].append(relation)
+        else:
+            self.relations_ids_by_entity_id[relation.source_entity_id] = [relation]
+
+        if relation.target_entity_id in self.relations_ids_by_entity_id:
+            self.relations_ids_by_entity_id[relation.target_entity_id].append(relation)
+        else:
+            self.relations_ids_by_entity_id[relation.target_entity_id] = [relation]
+
+        for nk in relation.pm_map:
+            self.pm_objects_nks_used.add(nk)
+
+    def register_dependency(self, dependent: Entity, dependency: Attribute) -> None:
+        if dependent.id in self.dependencies:
+            self.dependencies[dependent.id].add(dependency.id)
+        else:
+            self.dependencies[dependent.id] = {dependency.id}
+
+
+class LogicalModelsDatabase:
+    def __init__(self):
+        self.logical_models: Dict[str, LogicalModel] = {}
+
+    def register_logical_model(self, logical_model: LogicalModel) -> None:
+        self.logical_models[logical_model.version] = logical_model

dg_kit/base/physical_model.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import Dict
+
+from dg_kit.base.dataclasses.physical_model import (
+    Table,
+    Column,
+    Layer,
+)
+
+
+class PhysicalModel:
+    def __init__(self, version):
+        self.version = version
+        self.layers: Dict[str, Layer] = {}
+        self.tables: Dict[str, Table] = {}
+        self.columns: Dict[str, Column] = {}
+        self.dependencies: dict[str, set[str]] = {}
+        self.all_units_by_id: Dict[str, Layer | Table | Column] = {}
+        self.all_units_by_natural_key: Dict[str, Layer | Table | Column] = {}
+
+    def register_layer(self, layer: Layer):
+        self.layers[layer.id] = layer
+        self.all_units_by_id[layer.id] = layer
+        self.all_units_by_natural_key[layer.natural_key] = layer
+
+    def register_table(self, table: Table) -> None:
+        self.tables[table.id] = table
+        self.all_units_by_id[table.id] = table
+        self.all_units_by_natural_key[table.natural_key] = table
+
+    def register_column(self, column: Column) -> None:
+        self.columns[column.id] = column
+        self.all_units_by_id[column.id] = column
+        self.all_units_by_natural_key[column.natural_key] = column
+
+    def register_dependency(self, dependent: Table, dependency: Table) -> None:
+        if dependent.id in self.dependencies:
+            self.dependencies[dependent.id].add(dependency.id)
+        else:
+            self.dependencies[dependent.id] = {dependency.id}

dg_kit/integrations/dbt/README.md

@@ -0,0 +1,27 @@
+## dbt Integration
+
+Parses a dbt project into a `PhysicalModel` with layers, tables, columns, and dependencies.
+
+This integration is useful for extracting governance metadata from dbt and feeding it into
+CI checks, documentation, or data catalog syncs.
+
+## Requirements
+- A dbt project directory with `dbt_project.yml`
+- A `models/` directory containing model `.yml` and `.sql` files
+
+## Usage
+```python
+from dg_kit.integrations.dbt.parser import DBTParser
+
+pm = DBTParser("path/to/dbt_project").parse_pm()
+
+print(pm.version, len(pm.layers), len(pm.tables))
+```
+
+## What It Parses
+- Sources defined in top-level `models/*.yml`
+- Models and columns from layer `models/<layer>/*.yml`
+- Dependencies from `ref()` and `source()` calls in `models/**/*.sql`
+
+## Notes
+- Dependencies are registered only when referenced models/sources exist in the parsed metadata.