datacoolie-studio 0.1.1__py3-none-any.whl

datacoolie_studio/__init__.py

@@ -0,0 +1,3 @@
+"""DataCoolie Studio package."""
+
+__version__ = "0.1.0"

datacoolie_studio/__main__.py

@@ -0,0 +1,4 @@
+from datacoolie_studio.cli import main
+
+if __name__ == "__main__":
+    main()

datacoolie_studio/api/__init__.py

@@ -0,0 +1 @@
+

datacoolie_studio/api/v1/__init__.py

@@ -0,0 +1,3 @@
+from datacoolie_studio.api.v1.router import api_router
+
+__all__ = ["api_router"]

datacoolie_studio/api/v1/contracts/__init__.py

@@ -0,0 +1 @@
+"""Router-aligned API contracts."""

datacoolie_studio/api/v1/contracts/assets.py

@@ -0,0 +1,332 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+from datacoolie_studio.api.v1.contracts.lineage import SourceLocationResponse
+from datacoolie_studio.api.v1.contracts.shared import ReferenceResolutionResponse, ReferenceType, TargetIdentifierKind
+from datacoolie_studio.api.v1.contracts.workspace import ProjectReferenceMappingRead
+
+class AssetSummaryResponse(BaseModel):
+    assets: int
+    references: int = 0
+    manual_mappings: int = 0
+    visible: int = 0
+    asset_attention: int = 0
+    with_attention: int = 0
+    automatic_references: int = 0
+    manual_references: int = 0
+    unresolved_references: int = 0
+
+
+class AssetMetadataSourceResponse(BaseModel):
+    id: int
+    uri: str
+
+
+class AssetAttentionResponse(BaseModel):
+    severity: Literal["info", "warning", "error"]
+    code: str
+    message: str
+    source_type: str
+    subject_type: str = "asset"
+    dataflow_id: str | None = None
+    metadata_source_id: int | None = None
+    reference_id: str | None = None
+    reference_occurrence_id: str | None = None
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class AssetResponse(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str
+    display_name: str
+    friendly_name: str
+    full_identity: str
+    asset_type: Literal["table", "path", "sql_query", "python_function", "api", "unresolved"]
+    format: str | None = None
+    connection_name: str | None = None
+    connection_type: str | None = None
+    catalog: str | None = None
+    database: str | None = None
+    schema_name: str | None = None
+    table: str | None = None
+    path: str | None = None
+    query: str | None = None
+    python_function: str | None = None
+    roles: list[str] = Field(default_factory=list)
+    metadata_source_ids: list[int] = Field(default_factory=list)
+    metadata_sources: list[AssetMetadataSourceResponse] = Field(default_factory=list)
+    upstream_count: int
+    downstream_count: int
+    input_dataflow_count: int
+    output_dataflow_count: int
+    depends_on_count: int
+    used_by_count: int = 0
+    attention_count: int
+    attention_items: list[AssetAttentionResponse] = Field(default_factory=list)
+    identifiers: list[dict[str, Any]] = Field(default_factory=list)
+    observations: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class AssetFilterOptionsResponse(BaseModel):
+    connections: list[str] = Field(default_factory=list)
+    formats: list[str] = Field(default_factory=list)
+    asset_types: list[str] = Field(default_factory=list)
+    roles: list[str] = Field(default_factory=list)
+    attention_states: list[str] = Field(default_factory=list)
+
+
+class AssetReferenceGroupResponse(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str
+    reference_type: ReferenceType
+    normalized_value: str
+    display_name: str
+    resolution: ReferenceResolutionResponse
+    resolved_asset_id: str | None = None
+    resolved_asset_ids: list[str] = Field(default_factory=list)
+    resolved_asset: dict[str, Any] | None = None
+    candidate_asset_ids: list[str] = Field(default_factory=list)
+    candidate_assets: list[dict[str, Any]] = Field(default_factory=list)
+    occurrence_ids: list[str] = Field(default_factory=list)
+    consumer_asset_ids: list[str] = Field(default_factory=list)
+    consumer_assets: list[dict[str, Any]] = Field(default_factory=list)
+    provenances: list[str] = Field(default_factory=list)
+    resolution_methods: list[str] = Field(default_factory=list)
+    occurrence_count: int | None = None
+    dependency_count: int = 0
+    dataflow_ids: list[str] = Field(default_factory=list)
+    attention_count: int = 0
+    attention_items: list[AssetAttentionResponse] = Field(default_factory=list)
+    observations: list[dict[str, Any]] = Field(default_factory=list)
+    manual_mapping: dict[str, Any] | None = None
+
+
+class AssetReferenceOccurrenceResponse(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str
+    reference_id: str
+    reference_type: ReferenceType
+    raw_value: str
+    normalized_value: str
+    context_scope: str | None = None
+    context_scope_source: Literal["detected", "metadata_context"] | None = None
+    source_location: SourceLocationResponse | None = None
+    display_name: str
+    provenance: Literal["sql", "python", "python_sql"] | None = None
+    consumer_asset_id: str | None = None
+    consumer_asset: dict[str, Any] | None = None
+    connection_name: str | None = None
+    resolution: ReferenceResolutionResponse
+    resolution_method: str | None = None
+    resolved_asset_id: str | None = None
+    resolved_asset: dict[str, Any] | None = None
+    candidate_asset_ids: list[str] = Field(default_factory=list)
+    candidate_assets: list[dict[str, Any]] = Field(default_factory=list)
+    dependency_count: int = 0
+    dataflow_ids: list[str] = Field(default_factory=list)
+    attention_count: int = 0
+    attention_items: list[AssetAttentionResponse] = Field(default_factory=list)
+    observations: list[dict[str, Any]] = Field(default_factory=list)
+    manual_mapping: dict[str, Any] | None = None
+
+
+class AssetListItemResponse(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str
+    display_name: str
+    friendly_name: str
+    full_identity: str
+    asset_type: Literal["table", "path", "sql_query", "python_function", "api", "unresolved"]
+    format: str | None = None
+    connection_name: str | None = None
+    connection_type: str | None = None
+    catalog: str | None = None
+    database: str | None = None
+    schema_name: str | None = None
+    table: str | None = None
+    path: str | None = None
+    query: str | None = None
+    python_function: str | None = None
+    roles: list[str] = Field(default_factory=list)
+    metadata_source_ids: list[int] = Field(default_factory=list)
+    upstream_count: int
+    downstream_count: int
+    input_dataflow_count: int
+    output_dataflow_count: int
+    depends_on_count: int
+    used_by_count: int = 0
+    attention_count: int
+    identifier_count: int
+    observation_count: int
+    metadata_source_count: int
+    mapping_target: dict[str, str] | None = None
+
+
+class AssetInventoryResponse(BaseModel):
+    summary: AssetSummaryResponse
+    items: list[AssetListItemResponse]
+    filter_options: AssetFilterOptionsResponse
+    catalog_version: str
+
+
+class AssetReferenceListResponse(BaseModel):
+    items: list[AssetReferenceGroupResponse]
+    filter_options: dict[str, list[str]] = Field(default_factory=dict)
+    catalog_version: str
+
+
+class ProjectReferenceRegistryTargetResponse(BaseModel):
+    id: str
+    asset_id: str
+    asset_ids: list[str] = Field(default_factory=list)
+    environment_ids: list[int] = Field(default_factory=list)
+    environment_names: list[str] = Field(default_factory=list)
+    asset_type: Literal["table", "path", "sql_query", "python_function", "api", "unresolved"]
+    format: str | None = None
+    connection_name: str
+    display_name: str
+    context: str | None = None
+    kind: TargetIdentifierKind
+    value: str
+    display: str
+
+
+class ProjectReferenceRegistryEnvironmentResponse(BaseModel):
+    environment_id: int
+    environment_name: str
+    resolution: ReferenceResolutionResponse
+    resolved_asset_id: str | None = None
+    resolved_asset_ids: list[str] = Field(default_factory=list)
+    observed_target_ids: list[str] = Field(default_factory=list)
+    candidate_asset_ids: list[str] = Field(default_factory=list)
+    manual_mapping_id: int | None = None
+    manual_mapping_status: str | None = None
+    occurrence_count: int = 0
+    consumer_count: int = 0
+
+
+class ProjectReferenceRegistryObservedTargetResponse(BaseModel):
+    target: ProjectReferenceRegistryTargetResponse
+    environment_ids: list[int] = Field(default_factory=list)
+    environment_names: list[str] = Field(default_factory=list)
+
+
+class ProjectReferenceRegistryTargetCoverageResponse(BaseModel):
+    available_environment_names: list[str] = Field(default_factory=list)
+    missing_environment_names: list[str] = Field(default_factory=list)
+    available: int
+    total: int
+
+
+class ProjectReferenceRegistryRowResponse(BaseModel):
+    id: str
+    reference_type: ReferenceType
+    normalized_value: str
+    mapping: ProjectReferenceMappingRead | None = None
+    resolution: ReferenceResolutionResponse
+    environments: list[ProjectReferenceRegistryEnvironmentResponse] = Field(default_factory=list)
+    candidate_asset_ids: list[str] = Field(default_factory=list)
+    resolved_asset_ids: list[str] = Field(default_factory=list)
+    target: ProjectReferenceRegistryTargetResponse | None = None
+    observed_targets: list[ProjectReferenceRegistryObservedTargetResponse] = Field(default_factory=list)
+    target_coverage: ProjectReferenceRegistryTargetCoverageResponse
+
+
+class ProjectReferenceRegistryFailureResponse(BaseModel):
+    environment_id: int
+    environment_name: str
+    message: str
+
+
+class ProjectReferenceRegistryResponse(BaseModel):
+    project_id: int
+    mappings: list[ProjectReferenceMappingRead]
+    rows: list[ProjectReferenceRegistryRowResponse]
+    targets: list[ProjectReferenceRegistryTargetResponse]
+    failures: list[ProjectReferenceRegistryFailureResponse]
+
+
+class AssetReferenceDetailResponse(BaseModel):
+    reference: AssetReferenceGroupResponse
+    occurrences: list[AssetReferenceOccurrenceResponse] = Field(default_factory=list)
+    catalog_version: str
+
+
+class AssetSourceResponse(BaseModel):
+    definition: "AssetDefinitionResponse"
+    catalog_version: str
+
+
+class AssetDefinitionDiagnosticResponse(BaseModel):
+    severity: Literal["info", "warning", "error"]
+    code: str
+    message: str
+    details: dict[str, Any] = Field(default_factory=dict)
+
+
+class AssetDefinitionResponse(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    kind: Literal["sql_query", "python_function", "api", "path", "unresolved"]
+    language: str | None = None
+    status: Literal["available", "unavailable", "ambiguous", "empty"]
+    title: str | None = None
+    raw: str | None = None
+    formatted: str | None = None
+    source: str | None = None
+    function_path: str | None = None
+    module_name: str | None = None
+    relative_path: str | None = None
+    line_count: int = 0
+    diagnostics: list[AssetDefinitionDiagnosticResponse] = Field(default_factory=list)
+
+
+class AssetDetailResponse(BaseModel):
+    asset: AssetResponse
+    definition: AssetDefinitionResponse | None = None
+    attention_items: list[AssetAttentionResponse] = Field(default_factory=list)
+    direct_relationships: dict[str, Any] = Field(default_factory=dict)
+    upstream_assets: list[dict[str, Any]] = Field(default_factory=list)
+    downstream_assets: list[dict[str, Any]] = Field(default_factory=list)
+    input_flows: list[dict[str, Any]] = Field(default_factory=list)
+    output_flows: list[dict[str, Any]] = Field(default_factory=list)
+    depends_on: list[dict[str, Any]] = Field(default_factory=list)
+    used_by: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class ReferenceSourceMatchResponse(BaseModel):
+    line: int
+    column: int
+    end_line: int
+    end_column: int
+    precision: Literal["exact_reference", "detection_expression", "location_only"]
+
+
+class ReferenceSourceViewResponse(BaseModel):
+    id: Literal["query_source", "consumer_source", "evaluated_sql"]
+    label: str
+    language: Literal["sql", "python"]
+    content: str
+    path: str | None = None
+    function_path: str | None = None
+    module_name: str | None = None
+    matches: list[ReferenceSourceMatchResponse] = Field(default_factory=list)
+
+
+class ReferenceSourceDiagnosticResponse(BaseModel):
+    severity: Literal["info", "warning", "error"]
+    code: str
+    message: str
+
+
+class ReferenceOccurrenceSourceResponse(BaseModel):
+    occurrence_id: str
+    consumer_asset_id: str
+    views: list[ReferenceSourceViewResponse] = Field(default_factory=list)
+    diagnostics: list[ReferenceSourceDiagnosticResponse] = Field(default_factory=list)

datacoolie_studio/api/v1/contracts/credentials.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+CredentialProvider = Literal["s3", "minio", "adls", "onelake", "gcs", "dbfs"]
+
+
+class CredentialProfileCreateRequest(BaseModel):
+    name: str = Field(min_length=1, max_length=255)
+    provider: CredentialProvider
+    auth_type: str = Field(min_length=1, max_length=50)
+    config: dict[str, Any] = Field(default_factory=dict)
+    secret: dict[str, Any] | None = None
+
+
+class CredentialProfileUpdateRequest(BaseModel):
+    name: str | None = Field(default=None, min_length=1, max_length=255)
+    config: dict[str, Any] | None = None
+    secret: dict[str, Any] | None = None
+
+
+class CredentialProfileInfo(BaseModel):
+    id: str
+    name: str
+    provider: CredentialProvider
+    auth_type: str
+    secret_state: Literal["present", "missing", "unavailable"]
+    masked_summary: dict[str, Any]
+    version: int
+    reference_count: int
+    created_at: datetime
+    updated_at: datetime
+
+
+class CredentialProfileDetail(CredentialProfileInfo):
+    config: dict[str, Any]
+
+
+class CredentialCapabilities(BaseModel):
+    providers: dict[str, list[str]]
+    secret_store_available: bool
+    secret_store_backend: str
+    remediation: str | None = None

datacoolie_studio/api/v1/contracts/lineage.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, field_validator
+from datacoolie_studio.api.v1.contracts.shared import ReferenceResolutionResponse, ReferenceType
+
+class LineageSummaryResponse(BaseModel):
+    assets: int
+    references: int
+    dataflows: int
+    dependencies: int
+    stitched_assets: int
+    declared_assets: int
+    automatic_references: int
+    manual_references: int
+    unresolved_references: int
+    automatic_dependencies: int
+    manual_dependencies: int
+    unresolved_dependencies: int
+    diagnostics: int
+
+
+class LineageAssetResponse(BaseModel):
+    id: str
+    entity_type: Literal["asset"]
+    asset_type: Literal["table", "path", "sql_query", "python_function", "api", "unresolved"]
+    declaration_status: Literal["declared", "discovered_only"]
+    display_name: str
+    label: str
+    display_label: str | None = None
+    endpoint_locator: str | None = None
+    endpoint_kind: str | None = None
+    identity_type: str | None = None
+    connection_name: str | None = None
+    connection_type: str | None = None
+    format: str | None = None
+    catalog: str | None = None
+    database: str | None = None
+    schema_name: str | None = None
+    table: str | None = None
+    path: str | None = None
+    python_function: str | None = None
+    metadata_source_ids: list[int] = Field(default_factory=list)
+    roles: list[str] = Field(default_factory=list)
+    connection_names: list[str] = Field(default_factory=list)
+    mapping_target: dict[str, str] | None = None
+
+
+class LineageReferenceResponse(BaseModel):
+    id: str
+    entity_type: Literal["reference"]
+    reference_type: ReferenceType
+    display_name: str
+    normalized_value: str
+    resolution: ReferenceResolutionResponse
+    resolved_asset_id: str | None = None
+    resolved_asset_ids: list[str] = Field(default_factory=list)
+    candidate_asset_ids: list[str] = Field(default_factory=list)
+    occurrence_count: int = 0
+    consumer_asset_ids: list[str] = Field(default_factory=list)
+    provenances: list[Literal["sql", "python", "python_sql"]] = Field(default_factory=list)
+    dependency_count: int = 0
+    manual_mapping: dict[str, Any] | None = None
+
+    @field_validator("reference_type", mode="before")
+    @classmethod
+    def normalize_reference_type(cls, value: Any) -> Any:
+        if str(value or "").strip().lower() == "dynamic_expression":
+            return "unknown"
+        return value
+
+
+class SourceLocationResponse(BaseModel):
+    module: str | None = None
+    path: str | None = None
+    function_path: str | None = None
+    line: int | None = None
+    column: int | None = None
+    end_line: int | None = None
+    end_column: int | None = None
+    coordinate_space: Literal["query_source", "function_source"] | None = None
+
+
+class LineageDataflowResponse(BaseModel):
+    id: str
+    dataflow_id: str
+    name: str
+    source_asset_id: str
+    destination_asset_id: str
+    stage: str | None = None
+    load_type: str | None = None
+    metadata_source_id: int
+    metadata_source_uri: str
+
+
+class LineageDependencyResponse(BaseModel):
+    id: str
+    target_asset_id: str
+    consumer_asset_id: str
+    kind: Literal["reads", "uses"]
+    provenance: Literal["sql", "python", "python_sql"]
+    resolution: ReferenceResolutionResponse
+    resolution_method: str
+    reference_id: str
+    reference_occurrence_id: str
+    resolved_asset_id: str | None = None
+
+
+class LineageResponse(BaseModel):
+    schema_version: Literal["lineage.v4"]
+    summary: LineageSummaryResponse
+    assets: list[LineageAssetResponse]
+    references: list[LineageReferenceResponse]
+    dataflows: list[LineageDataflowResponse]
+    dependencies: list[LineageDependencyResponse]

datacoolie_studio/api/v1/contracts/metadata.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal
+
+from pydantic import BaseModel
+
+class MetadataResponse(BaseModel):
+    summary: dict[str, Any]
+    sources: list[dict[str, Any]]
+    connections: list[dict[str, Any]]
+    dataflows: list[dict[str, Any]]
+    schema_hints: list[dict[str, Any]]
+    errors: list[dict[str, Any]]
+
+
+class MetadataEditorDocumentResponse(BaseModel):
+    source: dict[str, Any]
+    sheets: dict[str, Any]
+    issues: list[dict[str, Any]]
+
+
+class MetadataEditorWorkspaceResponse(BaseModel):
+    schema_version: Literal["metadata-editor-workspace.v1"]
+    environment_id: int
+    metadata_catalog_version: str
+    document: MetadataEditorDocumentResponse
+    draft: MetadataEditorDocumentResponse | None = None
+
+
+class MetadataEditorValidationRequest(BaseModel):
+    source: dict[str, Any] | None = None
+    sheets: dict[str, Any]
+    issues: list[dict[str, Any]] = []
+
+
+class MetadataEditorValidationResponse(BaseModel):
+    status: str
+    summary: dict[str, Any]
+    issues: list[dict[str, Any]]
+
+
+class MetadataEditorSaveRequest(BaseModel):
+    expected_revision: dict[str, Any]
+    editor_document: MetadataEditorValidationRequest
+    confirm_overwrite: bool = False
+
+
+class MetadataBackupResponse(BaseModel):
+    id: int
+    project_id: int
+    environment_id: int
+    source_id: int
+    source_uri: str
+    backup_path: str
+    source_revision: dict[str, Any] | None = None
+    saved_revision: dict[str, Any] | None = None
+    created_at: datetime
+
+
+class MetadataBackupRestoreRequest(BaseModel):
+    expected_revision: dict[str, Any]
+    confirm_restore: bool = False

datacoolie_studio/api/v1/contracts/modules.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel
+
+class ModuleInfo(BaseModel):
+    key: str
+    name: str
+    description: str
+    group: str
+    status: Literal["available", "coming_soon"]
+    togglable: bool
+    default_enabled: bool
+    pages: list[str]
+    enabled: bool
+
+
+class ModuleStateUpdateRequest(BaseModel):
+    enabled: bool