modelable 1.0.0__py3-none-any.whl

modelable/governance/checker.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+from modelable.expressions.cel import extract_field_refs, parse_cel
+from modelable.parser.ir import (
+    AccessBlock,
+    ClassificationLevel,
+    ComputedMapping,
+    DirectMapping,
+    FieldDef,
+    MdlFile,
+    ModelVersion,
+    ProjectionField,
+    ProjectionVersion,
+)
+from modelable.registry.resolver import resolve_model_ref
+
+
+@dataclass(frozen=True)
+class GovernanceFinding:
+    code: str
+    subject: str
+    message: str
+
+    def as_dict(self) -> dict[str, str]:
+        return asdict(self)
+
+
+def build_projection_governance_findings(
+    domain_name: str,
+    projection_name: str,
+    pv: ProjectionVersion,
+    mdl: MdlFile,
+) -> list[GovernanceFinding]:
+    findings: list[GovernanceFinding] = []
+    subject = f"{domain_name}.{projection_name}@{pv.version}"
+
+    _check_projection_access(subject, domain_name, pv.access, findings)
+    _check_computed_field_access(subject, domain_name, pv, mdl, findings)
+    _check_projection_classification(subject, pv, mdl, findings)
+
+    return findings
+
+
+def _check_projection_access(
+    subject: str,
+    domain_name: str,
+    access: AccessBlock | None,
+    findings: list[GovernanceFinding],
+) -> None:
+    if access is None:
+        findings.append(
+            GovernanceFinding(
+                code="missing_project_grant",
+                subject=subject,
+                message=f"{subject} has no documented project grant",
+            )
+        )
+        findings.append(
+            GovernanceFinding(
+                code="missing_read_grant",
+                subject=subject,
+                message=f"{subject} has no documented read grant",
+            )
+        )
+        return
+
+    if not _has_entity_permission(access, "project"):
+        findings.append(
+            GovernanceFinding(
+                code="missing_project_grant",
+                subject=subject,
+                message=f"{subject} has no documented project grant for domain '{domain_name}'",
+            )
+        )
+
+    if not _has_entity_permission(access, "read"):
+        findings.append(
+            GovernanceFinding(
+                code="missing_read_grant",
+                subject=subject,
+                message=f"{subject} has no documented read grant for domain '{domain_name}'",
+            )
+        )
+
+
+def _check_computed_field_access(
+    subject: str,
+    projection_domain: str,
+    pv: ProjectionVersion,
+    mdl: MdlFile,
+    findings: list[GovernanceFinding],
+) -> None:
+    resolved_sources = _build_resolved_sources(pv, mdl)
+
+    for proj_field, source_refs in _iter_projection_field_sources(pv, resolved_sources):
+        if not isinstance(proj_field.mapping, ComputedMapping):
+            continue
+
+        for target_ref, target_version, field_name in source_refs:
+            if not _has_property_derivation(target_version.access, field_name, projection_domain):
+                findings.append(
+                    GovernanceFinding(
+                        code="missing_derivation_policy",
+                        subject=subject,
+                        message=(
+                            f"{subject}.{proj_field.name} uses source field "
+                            f"{target_ref}.{field_name} without documented derivation policy"
+                        ),
+                    )
+                )
+
+
+def _check_projection_classification(
+    subject: str,
+    pv: ProjectionVersion,
+    mdl: MdlFile,
+    findings: list[GovernanceFinding],
+) -> None:
+    resolved_sources = _build_resolved_sources(pv, mdl)
+
+    for proj_field, source_refs in _iter_projection_field_sources(pv, resolved_sources):
+        for target_ref, target_version, field_name in source_refs:
+            source_field = _find_field(target_version, field_name)
+            if source_field is None:
+                continue
+
+            _check_pii_preservation(subject, proj_field, target_ref, source_field, findings)
+            _check_classification_preservation(subject, proj_field, target_ref, source_field, findings)
+
+
+def _has_entity_permission(access: AccessBlock, permission: str) -> bool:
+    return any(permission in grant.permissions for grant in access.entity)
+
+
+def _has_property_derivation(
+    access: AccessBlock | None,
+    field_name: str,
+    projection_domain: str,
+) -> bool:
+    if access is None:
+        return False
+
+    grants = access.properties.get(field_name, [])
+    for grant in grants:
+        if grant.principal != projection_domain:
+            continue
+        if "read" in grant.permissions and "derive" in grant.permissions:
+            return True
+    return False
+
+
+def _iter_projection_field_sources(
+    pv: ProjectionVersion,
+    resolved_sources: dict[str, tuple[str, ModelVersion]],
+) -> list[tuple[ProjectionField, list[tuple[str, ModelVersion, str]]]]:
+    result: list[tuple[ProjectionField, list[tuple[str, ModelVersion, str]]]] = []
+    for proj_field in pv.fields:
+        source_refs: list[tuple[str, ModelVersion, str]] = []
+        mapping = proj_field.mapping
+
+        if isinstance(mapping, DirectMapping):
+            target = resolved_sources.get(mapping.source_alias)
+            if target is not None:
+                target_ref, target_version = target
+                source_refs.append((target_ref, target_version, mapping.source_field))
+        elif isinstance(mapping, ComputedMapping):
+            expr_ast, _ = parse_cel(mapping.expression)
+            if expr_ast is not None:
+                for alias, field_name in extract_field_refs(expr_ast):
+                    target = resolved_sources.get(alias)
+                    if target is None:
+                        continue
+                    target_ref, target_version = target
+                    source_refs.append((target_ref, target_version, field_name))
+
+        result.append((proj_field, source_refs))
+
+    return result
+
+
+def _check_pii_preservation(
+    subject: str,
+    proj_field: ProjectionField,
+    source_ref: str,
+    source_field: FieldDef,
+    findings: list[GovernanceFinding],
+) -> None:
+    if not source_field.is_pii or proj_field.is_pii:
+        return
+
+    findings.append(
+        GovernanceFinding(
+            code="missing_pii_metadata",
+            subject=subject,
+            message=(
+                f"{subject}.{proj_field.name} projects {source_ref}.{source_field.name} "
+                "without preserving @pii metadata"
+            ),
+        )
+    )
+
+
+def _check_classification_preservation(
+    subject: str,
+    proj_field: ProjectionField,
+    source_ref: str,
+    source_field: FieldDef,
+    findings: list[GovernanceFinding],
+) -> None:
+    source_level = source_field.classification
+    if source_level is None:
+        return
+
+    projected_level = proj_field.classification
+    if projected_level is None:
+        findings.append(
+            GovernanceFinding(
+                code="missing_classification_metadata",
+                subject=subject,
+                message=(
+                    f"{subject}.{proj_field.name} projects {source_ref}.{source_field.name} "
+                    "without preserving classification metadata"
+                ),
+            )
+        )
+        return
+
+    if _classification_rank(projected_level) < _classification_rank(source_level):
+        findings.append(
+            GovernanceFinding(
+                code="lowered_classification",
+                subject=subject,
+                message=(
+                    f"{subject}.{proj_field.name} lowers classification from "
+                    f"{source_level.value} to {projected_level.value}"
+                ),
+            )
+        )
+
+
+def _classification_rank(level: ClassificationLevel) -> int:
+    order = {
+        ClassificationLevel.open: 0,
+        ClassificationLevel.internal: 1,
+        ClassificationLevel.confidential: 2,
+        ClassificationLevel.restricted: 3,
+        ClassificationLevel.secret: 4,
+    }
+    return order[level]
+
+
+def _find_field(version: ModelVersion, field_name: str) -> FieldDef | None:
+    return next((field for field in version.fields if field.name == field_name), None)
+
+
+def _build_resolved_sources(pv: ProjectionVersion, mdl: MdlFile) -> dict[str, tuple[str, ModelVersion]]:
+    resolved_sources: dict[str, tuple[str, ModelVersion]] = {}
+    all_sources = [(pv.source.model, pv.source.version, pv.source.alias)]
+    for join in pv.joins:
+        all_sources.append((join.model, join.version, join.alias))
+
+    for model_ref, version_spec, alias in all_sources:
+        try:
+            resolved = resolve_model_ref(mdl, model_ref, version_spec)
+            resolved_sources[alias] = (f"{model_ref}@{resolved.version.version}", resolved.version)
+        except LookupError:
+            pass
+
+    return resolved_sources

modelable/governance/por.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+DEFAULT_POR_ISSUER = "modelable-registry.local"
+DEFAULT_POR_ISSUED_AT = "1970-01-01T00:00:00Z"
+
+
+@dataclass(frozen=True)
+class PortableOwnershipRecord:
+    model: str
+    issuer: str = DEFAULT_POR_ISSUER
+    issued_at: str = DEFAULT_POR_ISSUED_AT
+    signature: str | None = None
+
+    def as_dict(self) -> dict[str, str | None]:
+        return {
+            "model": self.model,
+            "issuer": self.issuer,
+            "issuedAt": self.issued_at,
+            **({"signature": self.signature} if self.signature else {}),
+        }
+
+
+def build_por_reference(ref: str, *, issuer: str = DEFAULT_POR_ISSUER) -> dict[str, str]:
+    """Return the unsigned POR reference embedded in generated artifacts."""
+    return {
+        "model": ref,
+        "issuer": issuer,
+        "issuedAt": DEFAULT_POR_ISSUED_AT,
+    }
+
+
+def build_por_record(
+    ref: str,
+    *,
+    issuer: str = DEFAULT_POR_ISSUER,
+    issued_at: str = DEFAULT_POR_ISSUED_AT,
+    signature: str | None = None,
+) -> PortableOwnershipRecord:
+    return PortableOwnershipRecord(
+        model=ref,
+        issuer=issuer,
+        issued_at=issued_at,
+        signature=signature,
+    )

modelable/grammar/__init__.py

@@ -0,0 +1 @@
+

modelable/grammar/modelable.lark

@@ -0,0 +1,257 @@
+// Modelable IDL Grammar
+// Parser: Earley
+
+start: statement+
+
+statement: domain_decl
+         | binding_decl
+         | workspace_decl
+         | import_domain_stmt
+         | projection_decl
+         | subscription_decl
+         | consumer_decl
+
+
+// -- Domain ------------------------------------------------------------------
+
+domain_decl: "domain" domain_name "{" domain_item* "}"
+domain_name: IDENT | ESCAPED_STRING
+
+domain_item: owner_attr
+           | contact_attr
+           | desc_attr
+           | model_decl
+           | projection_decl
+           | auto_projections_decl
+           | generate_block
+
+owner_attr: "owner" ":" ESCAPED_STRING
+contact_attr: "contact" ":" ESCAPED_STRING
+desc_attr: "description" ":" ESCAPED_STRING
+
+
+// -- Model -------------------------------------------------------------------
+
+model_decl: wire_annotation* model_kind IDENT model_header? "{" model_body_item* "}"
+model_header: "@" INT model_change?
+            | model_change
+model_change: "(" change_kind ")"
+
+model_body_item: field_decl
+               | access_block
+
+model_kind: "entity"    -> mk_entity
+          | "aggregate" -> mk_aggregate
+          | "event"     -> mk_event
+          | "value"     -> mk_value
+
+change_kind: "additive" -> ck_additive
+           | "breaking" -> ck_breaking
+
+field_decl: annotation* IDENT optional_marker? ":" type_expr field_default?
+optional_marker: "?"
+field_default: "=" EXPRESSION
+
+annotation: "@key"                                                -> ann_key
+          | "@pii"                                                -> ann_pii
+          | "@classification" "(" ESCAPED_STRING ")"             -> ann_classification
+          | "@deprecated" "(" "replacedBy" ":" ESCAPED_STRING ")" -> ann_deprecated
+          | "@owner" "(" ESCAPED_STRING ")"                      -> ann_owner
+          | "@server"                                             -> ann_server
+          | wire_annotation
+          | "@pitCutoff" "(" ANNOTATION_EXPR ")"                  -> ann_pit_cutoff
+          | "@latestBefore" "(" ANNOTATION_EXPR ")"               -> ann_latest_before
+          | "@latestOnly"                                         -> ann_latest_only
+
+wire_annotation: "@wire" "(" wire_option ("," wire_option)* ")" -> ann_wire
+wire_option: wire_key ":" wire_value
+wire_key: IDENT ("." IDENT)?
+wire_value: ESCAPED_STRING        -> wire_string
+          | wire_map
+wire_map: "{" wire_map_item ("," wire_map_item)* "}"
+wire_map_item: IDENT ":" ESCAPED_STRING
+
+
+// -- Types -------------------------------------------------------------------
+
+type_expr: primitive_type
+         | decimal_type
+         | enum_type
+         | array_type
+         | map_type
+         | ref_type
+         | object_type
+         | IDENT
+
+primitive_type: "string"    -> pt_string
+              | "int"       -> pt_int
+              | "float"     -> pt_float
+              | "bool"      -> pt_bool
+              | "date"      -> pt_date
+              | "time"      -> pt_time
+              | "timestamp" -> pt_timestamp
+              | "uuid"      -> pt_uuid
+              | "duration"  -> pt_duration
+              | "binary"    -> pt_binary
+              | "json"      -> pt_json
+
+decimal_type: "decimal" "(" INT "," INT ")"
+enum_type: "enum" "(" IDENT ("," IDENT)* ")"
+array_type: "array" "<" type_expr ">"
+map_type: "map" "<" type_expr "," type_expr ">"
+ref_type: "ref" "<" dotted_ref ">"
+object_type: "object" "{" field_decl* "}"
+
+
+// -- Projections --------------------------------------------------------------
+
+projection_decl: wire_annotation* "projection" IDENT "@" INT projection_source_block? source_clause? "{" projection_body_item* "}"
+projection_source_block: "{" source_clause "}"
+
+projection_body_item: proj_field
+                    | access_block
+                    | subscription_block
+                    | generate_block
+                    | materialisation_block
+
+source_clause: "from" dotted_ref "@" version_spec "as" IDENT join_clause* where_clause? group_clause?
+
+where_clause: "where" FIELD_EXPRESSION
+
+join_modifier: "left"
+
+join_clause: join_prefix annotation*
+          | join_prefix join_option*
+join_option: annotation
+           | cardinality_attr
+cardinality_attr: "cardinality" ":" IDENT
+join_prefix: "join" dotted_ref "@" version_spec "as" IDENT "on" EXPRESSION
+           | join_modifier "join" dotted_ref "@" version_spec "as" IDENT "on" EXPRESSION
+
+group_clause: "group" "by" group_item ("," group_item)*
+
+group_item: /[^,\n\r{}]+/
+
+version_spec: INT                  -> version_exact
+            | INT "#" IDENT       -> version_pinned
+            | ">=" INT "<" INT     -> version_range
+            | ">=" INT             -> version_min
+
+proj_field: annotation* IDENT "<-" qualified_field -> direct_field
+          | annotation* IDENT "=" FIELD_EXPRESSION  -> computed_field
+
+subscription_block: "subscription" "{" subscription_item* "}"
+subscription_item: IDENT ":" EXPRESSION
+materialisation_block: "materialisation" "{" materialisation_item* "}"
+materialisation_item: IDENT ":" EXPRESSION
+
+qualified_field: IDENT "." IDENT
+
+
+// -- Generate ----------------------------------------------------------------
+
+generate_block: "generate" "{" generate_target* "}"
+
+generate_target: target_name ("->" ESCAPED_STRING)?
+
+target_name: "openapi"                 -> tn_openapi
+           | "typescript"              -> tn_typescript
+           | "avro"                    -> tn_avro
+           | "protobuf"                -> tn_protobuf
+           | "sql" "(" db_dialect ")"  -> tn_sql
+           | "jsonschema"              -> tn_jsonschema
+           | "asyncapi"                -> tn_asyncapi
+           | "docs"                    -> tn_docs
+
+db_dialect: "postgres" -> dd_postgres
+          | "mysql"    -> dd_mysql
+          | "clickhouse" -> dd_clickhouse
+          | "sqlite"   -> dd_sqlite
+
+
+// -- Deferred top-level forms ------------------------------------------------
+// These are parsed more fully in later parser-plan tasks.
+
+binding_decl: "binding" IDENT "{" binding_item* "}"
+binding_item: binding_adapter_attr
+            | binding_model_attr
+            | binding_table_attr
+            | ignored_block_item
+binding_adapter_attr: "adapter" ":" IDENT
+binding_model_attr: "model" ":" dotted_ref "@" INT
+binding_table_attr: "table" ":" ESCAPED_STRING
+subscription_decl: "subscription" IDENT "{" subscription_decl_item* "}"
+subscription_decl_item: IDENT ":" EXPRESSION
+workspace_decl: "workspace" workspace_label? "{" workspace_item* "}"
+workspace_label: IDENT | ESCAPED_STRING
+workspace_item: workspace_name_attr
+              | workspace_description_attr
+              | registry_block
+              | peers_block
+              | generate_block
+              | ai_block
+workspace_name_attr: "name" ":" ESCAPED_STRING
+workspace_description_attr: "description" ":" ESCAPED_STRING
+registry_block: "registry" "{" ignored_block_item* "}"
+peers_block: "peers" ":" "[" PEERS_CONTENT "]"
+ai_block: "ai" "{" ai_item* "}"
+ai_item: ai_provider | ai_model | ai_repair_attempts
+ai_provider: "provider" ":" ESCAPED_STRING
+ai_model: "model" ":" ESCAPED_STRING
+ai_repair_attempts: "repair_attempts" ":" INT
+import_domain_stmt: "import" "domain" dotted_ref import_source? pinned_import? version_selector?
+import_source: "from" "registry" ESCAPED_STRING
+pinned_import: "at" dotted_ref "@" INT "#" IDENT
+consumer_decl: "consumer" IDENT? "{" ignored_block_item* "}"
+auto_projections_decl: "auto" "projections" IDENT "@" INT "{" auto_projection_item* "}"
+
+auto_projection_item: auto_projection_kind auto_projection_option*
+auto_projection_kind: "db"      -> apk_db
+                    | "request" -> apk_request
+                    | "reply"   -> apk_reply
+                    | "event"   -> apk_event
+
+auto_projection_option: exclude_option | on_option
+exclude_option: "exclude" "[" auto_projection_exclusion ("," auto_projection_exclusion)* "]"
+on_option: "on" "[" IDENT ("," IDENT)* "]"
+auto_projection_exclusion: annotation | IDENT
+
+access_block: "access" "{" access_item* "}"
+access_item: entity_grant
+           | property_grant
+entity_grant: "entity" principal permission_list
+property_grant: "property" IDENT principal permission_list
+principal: PRINCIPAL
+permission_list: "[" permission ("," permission)* "]"
+permission: "read" -> p_read
+          | "project" -> p_project
+          | "subscribe" -> p_subscribe
+          | "write" -> p_write
+          | "transfer" -> p_transfer
+          | "manage_access" -> p_manage_access
+          | "derive" -> p_derive
+          | "redact" -> p_redact
+
+version_selector: "@" version_expr
+version_expr: INT | VERSION_RANGE
+dotted_ref: IDENT ("." IDENT)*
+
+ignored_block_item: /[^{}]+/ | "{" ignored_block_item* "}"
+
+
+// -- Lexing ------------------------------------------------------------------
+
+VERSION_RANGE: /[<>~=^0-9.,\s]+/
+EXPRESSION: /[^\n\r{}]+/
+FIELD_EXPRESSION: /[^\n\r]+(?:\n(?!\s*(?:@|[A-Za-z_][A-Za-z0-9_-]*\s*(?:<-|=)|generate\s*\{|materialisation\s*\{|subscription\s*\{|access\s*\{|from\s|group\s+by|\{|\}))[^\n\r]+)*/
+ANNOTATION_EXPR: /[^)\n\r{}]+/
+IDENT: /[A-Za-z_][A-Za-z0-9_-]*/
+PEERS_CONTENT: /[^\]]+/
+PRINCIPAL: /\*|[A-Za-z_][A-Za-z0-9_.-]*/
+
+COMMENT: /\/\/[^\n]*/
+%import common.ESCAPED_STRING
+%import common.INT
+%import common.WS
+%ignore WS
+%ignore COMMENT

modelable/graph/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .export import build_graph_export, write_graph_export
+
+__all__ = ["build_graph_export", "write_graph_export"]