pinky-provider 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pinky_provider/__init__.py +26 -0
- pinky_provider/cli/__init__.py +3 -0
- pinky_provider/cli/main.py +84 -0
- pinky_provider/core/__init__.py +5 -0
- pinky_provider/core/diff.py +75 -0
- pinky_provider/core/manifest.py +111 -0
- pinky_provider/core/schemas.py +47 -0
- pinky_provider/core/session.py +56 -0
- pinky_provider/dag/__init__.py +3 -0
- pinky_provider/dag/dag.py +27 -0
- pinky_provider/provider.py +87 -0
- pinky_provider/resources/__init__.py +27 -0
- pinky_provider/resources/account/__init__.py +96 -0
- pinky_provider/resources/account/api_integration.py +6 -0
- pinky_provider/resources/account/compute_pool.py +6 -0
- pinky_provider/resources/account/database.py +6 -0
- pinky_provider/resources/account/external_access_integration.py +6 -0
- pinky_provider/resources/account/external_volume.py +6 -0
- pinky_provider/resources/account/network_policy.py +6 -0
- pinky_provider/resources/account/network_rule.py +6 -0
- pinky_provider/resources/account/notification_integration.py +6 -0
- pinky_provider/resources/account/role.py +6 -0
- pinky_provider/resources/account/secret.py +6 -0
- pinky_provider/resources/account/tag.py +6 -0
- pinky_provider/resources/account/user.py +6 -0
- pinky_provider/resources/account/warehouse.py +72 -0
- pinky_provider/resources/base.py +160 -0
- pinky_provider/resources/database/__init__.py +41 -0
- pinky_provider/resources/database/database_role.py +6 -0
- pinky_provider/resources/database/grant.py +6 -0
- pinky_provider/resources/database/schema.py +6 -0
- pinky_provider/resources/schema/__init__.py +68 -0
- pinky_provider/resources/schema/alert.py +6 -0
- pinky_provider/resources/schema/dynamic_table.py +6 -0
- pinky_provider/resources/schema/event_table.py +6 -0
- pinky_provider/resources/schema/iceberg_table.py +6 -0
- pinky_provider/resources/schema/procedure.py +6 -0
- pinky_provider/resources/schema/secret.py +6 -0
- pinky_provider/resources/schema/stage.py +6 -0
- pinky_provider/resources/schema/storage_lifecycle_policy.py +6 -0
- pinky_provider/resources/schema/stream.py +6 -0
- pinky_provider/resources/schema/streamlit.py +6 -0
- pinky_provider/resources/schema/table.py +6 -0
- pinky_provider/resources/schema/tag.py +6 -0
- pinky_provider/resources/schema/task.py +6 -0
- pinky_provider/resources/schema/user_defined_function.py +6 -0
- pinky_provider/resources/schema/view.py +6 -0
- pinky_provider-0.1.0.dev1.dist-info/METADATA +103 -0
- pinky_provider-0.1.0.dev1.dist-info/RECORD +52 -0
- pinky_provider-0.1.0.dev1.dist-info/WHEEL +4 -0
- pinky_provider-0.1.0.dev1.dist-info/entry_points.txt +2 -0
- pinky_provider-0.1.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""pinky-provider — declarative, idempotent IaC engine for Snowflake.
|
|
2
|
+
|
|
3
|
+
Top-level names are imported lazily (PEP 562): ``import pinky_provider`` stays light and never
|
|
4
|
+
pulls ``snowflake.core`` until you actually touch :class:`Provider` or
|
|
5
|
+
:class:`~pinky_provider.resources.schema.SchemaProvider`. This keeps the engine's pure logic
|
|
6
|
+
(``core.diff``, ``core.manifest``, ``resources.base``) importable on its own — and supports the
|
|
7
|
+
lightweight ``ARTIFACT_REPOSITORY`` import path (ADR-0014).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
__all__ = ["Provider", "SchemaProvider"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def __getattr__(name: str) -> Any:
|
|
18
|
+
if name == "Provider":
|
|
19
|
+
from pinky_provider.provider import Provider
|
|
20
|
+
|
|
21
|
+
return Provider
|
|
22
|
+
if name == "SchemaProvider":
|
|
23
|
+
from pinky_provider.resources.schema import SchemaProvider
|
|
24
|
+
|
|
25
|
+
return SchemaProvider
|
|
26
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import rich_click as click
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
|
|
6
|
+
from pinky_provider.core.diff import Action, ResourceDiff
|
|
7
|
+
|
|
8
|
+
# Symbol + colour per action, matching the plan/apply display contract (ADR-0003).
|
|
9
|
+
_STYLE: dict[Action, tuple[str, str]] = {
|
|
10
|
+
Action.CREATE: ("+", "green"),
|
|
11
|
+
Action.ALTER: ("~", "yellow"),
|
|
12
|
+
Action.UNCHANGED: ("=", "dim"),
|
|
13
|
+
Action.DROP: ("-", "red"),
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _render(console: Console, diffs: list[ResourceDiff], title: str) -> None:
|
|
18
|
+
console.print(f"[bold]{title}[/bold]")
|
|
19
|
+
if not diffs:
|
|
20
|
+
console.print(" [dim]no resources declared[/dim]")
|
|
21
|
+
return
|
|
22
|
+
for diff in diffs:
|
|
23
|
+
symbol, colour = _STYLE[diff.action]
|
|
24
|
+
detail = f" [dim]({diff.details})[/dim]" if diff.details else ""
|
|
25
|
+
console.print(
|
|
26
|
+
f" [{colour}]{symbol} {diff.resource_type}/{diff.name}[/{colour}]"
|
|
27
|
+
f" [{colour}]{diff.action}[/{colour}]{detail}"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@click.group()
|
|
32
|
+
def main() -> None:
|
|
33
|
+
"""pinky-provider — declarative IaC engine for Snowflake."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@main.command()
|
|
37
|
+
@click.option(
|
|
38
|
+
"--env", required=True, help="Target environment (SANDBOX or PRODUCTION)."
|
|
39
|
+
)
|
|
40
|
+
@click.option("--source", default="manifest.yml", show_default=True)
|
|
41
|
+
def plan(env: str, source: str) -> None:
|
|
42
|
+
"""Show changes without applying them."""
|
|
43
|
+
from pinky_provider.core.session import get_session
|
|
44
|
+
from pinky_provider.provider import Provider
|
|
45
|
+
|
|
46
|
+
console = Console()
|
|
47
|
+
provider = Provider(get_session())
|
|
48
|
+
diffs = provider.plan(source=source, env=env)
|
|
49
|
+
_render(console, diffs, f"Plan · {env}")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@main.command()
|
|
53
|
+
@click.option(
|
|
54
|
+
"--env", required=True, help="Target environment (SANDBOX or PRODUCTION)."
|
|
55
|
+
)
|
|
56
|
+
@click.option("--source", default="manifest.yml", show_default=True)
|
|
57
|
+
def apply(env: str, source: str) -> None:
|
|
58
|
+
"""Apply the manifest to the target schema."""
|
|
59
|
+
from pinky_provider.core.session import get_session
|
|
60
|
+
from pinky_provider.provider import Provider
|
|
61
|
+
|
|
62
|
+
console = Console()
|
|
63
|
+
provider = Provider(get_session())
|
|
64
|
+
diffs = provider.apply(source=source, env=env)
|
|
65
|
+
_render(console, diffs, f"Apply · {env}")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@main.command()
|
|
69
|
+
@click.option(
|
|
70
|
+
"--out", default=".pinky/schemas", show_default=True, help="Output directory."
|
|
71
|
+
)
|
|
72
|
+
def schemas(out: str) -> None:
|
|
73
|
+
"""Generate JSON Schemas for resource YAML files (editor autocomplete). No Snowflake session."""
|
|
74
|
+
from pinky_provider.core.schemas import write_schemas
|
|
75
|
+
|
|
76
|
+
console = Console()
|
|
77
|
+
written = write_schemas(out)
|
|
78
|
+
console.print(f"[bold]Schemas · {len(written)}[/bold]")
|
|
79
|
+
for type_name, path in written:
|
|
80
|
+
console.print(f" [green]+[/green] {type_name} [dim]{path}[/dim]")
|
|
81
|
+
console.print(
|
|
82
|
+
"\n[dim]Wire autocomplete — add to .vscode/settings.json (key = schema, value = glob):[/dim]\n"
|
|
83
|
+
f' "yaml.schemas": {{ "./{out}/<type>.schema.json": "resources/<type>/*.yml" }}'
|
|
84
|
+
)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable, Mapping
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from enum import StrEnum
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Action(StrEnum):
|
|
10
|
+
CREATE = "CREATE"
|
|
11
|
+
ALTER = "ALTER"
|
|
12
|
+
UNCHANGED = "UNCHANGED"
|
|
13
|
+
DROP = "DROP"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class ResourceDiff:
|
|
18
|
+
"""Result of comparing desired resource state against Snowflake's current state.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
resource_type: The resource type key (e.g. ``table``, ``warehouse``).
|
|
22
|
+
name: Object name.
|
|
23
|
+
action: The DDL action the provider will take.
|
|
24
|
+
details: Comma-separated changed field names (populated for ``ALTER`` only).
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
resource_type: str
|
|
28
|
+
name: str
|
|
29
|
+
action: Action
|
|
30
|
+
details: str = ""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _normalize(value: Any) -> Any:
|
|
34
|
+
"""Default comparison: case-insensitive for strings, pass-through otherwise."""
|
|
35
|
+
return value.upper() if isinstance(value, str) else value
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def compute_diff(
|
|
39
|
+
resource_type: str,
|
|
40
|
+
name: str,
|
|
41
|
+
desired: dict[str, Any],
|
|
42
|
+
actual: dict[str, Any] | None,
|
|
43
|
+
normalizers: Mapping[str, Callable[[Any], Any]] | None = None,
|
|
44
|
+
) -> ResourceDiff:
|
|
45
|
+
"""Compare a desired resource descriptor to the current Snowflake object state.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
resource_type: Resource type key.
|
|
49
|
+
name: Object name.
|
|
50
|
+
desired: The SDK-facing payload (``to_sdk_dict``); ``_``-keys and ``name`` are ignored.
|
|
51
|
+
actual: Current state fetched from ``snowflake.core`` (``.to_dict()``), or ``None`` if absent.
|
|
52
|
+
normalizers: Per-field comparison function (e.g. ``warehouse_size`` → size canonicaliser),
|
|
53
|
+
declared per resource via ``_field_normalizers``. Fields without one use case-insensitive
|
|
54
|
+
comparison.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
A :class:`ResourceDiff` describing the required DDL action. Only fields present in
|
|
58
|
+
``desired`` are compared — pinky never asserts on fields it does not declare.
|
|
59
|
+
"""
|
|
60
|
+
if actual is None:
|
|
61
|
+
return ResourceDiff(resource_type, name, Action.CREATE)
|
|
62
|
+
|
|
63
|
+
field_normalizers = normalizers or {}
|
|
64
|
+
changed = []
|
|
65
|
+
for key, want in desired.items():
|
|
66
|
+
if key.startswith("_") or key == "name":
|
|
67
|
+
continue
|
|
68
|
+
norm = field_normalizers.get(key, _normalize)
|
|
69
|
+
if norm(actual.get(key)) != norm(want):
|
|
70
|
+
changed.append(key)
|
|
71
|
+
if changed:
|
|
72
|
+
return ResourceDiff(
|
|
73
|
+
resource_type, name, Action.ALTER, ", ".join(sorted(changed))
|
|
74
|
+
)
|
|
75
|
+
return ResourceDiff(resource_type, name, Action.UNCHANGED)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import yaml
|
|
8
|
+
from jinja2 import Environment, StrictUndefined, UndefinedError
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class ResourceSpec:
|
|
13
|
+
"""One declared Snowflake object, after Jinja resolution.
|
|
14
|
+
|
|
15
|
+
Attributes:
|
|
16
|
+
resource_type: Type key, inferred from the parent folder (``resources/warehouse/`` → ``warehouse``).
|
|
17
|
+
data: Rendered YAML payload — Snowflake fields plus ``_``-prefixed provider keys.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
resource_type: str
|
|
21
|
+
data: dict[str, Any] = field(default_factory=dict)
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def name(self) -> str:
|
|
25
|
+
return str(self.data.get("name", ""))
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Manifest:
|
|
29
|
+
"""Parsed ``manifest.yml`` with Jinja2 variable resolution applied.
|
|
30
|
+
|
|
31
|
+
The manifest declares the deployment ``scope`` and the resource files to load. Each resource
|
|
32
|
+
file is rendered with the variables from ``vars/<ENV>.yml`` (overridable), then parsed. Its
|
|
33
|
+
type is the parent folder name — ``1 file = 1 object`` (AGENT.md).
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
source: Path to ``manifest.yml``.
|
|
37
|
+
env: Target environment — selects ``vars/<env>.yml`` (e.g. ``SANDBOX``).
|
|
38
|
+
vars: Variable overrides injected on top of the env file.
|
|
39
|
+
|
|
40
|
+
Attributes:
|
|
41
|
+
resources: Resource specs in declared order.
|
|
42
|
+
scope: ``schema``, ``database``, or ``account``.
|
|
43
|
+
env: The resolved environment name.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
resources: list[ResourceSpec]
|
|
47
|
+
scope: str
|
|
48
|
+
env: str
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self, source: str | Path, env: str, vars: dict[str, str] | None = None
|
|
52
|
+
) -> None:
|
|
53
|
+
source_path = Path(source).resolve()
|
|
54
|
+
self.root_dir = source_path.parent
|
|
55
|
+
self.env = env
|
|
56
|
+
|
|
57
|
+
manifest = yaml.safe_load(source_path.read_text()) or {}
|
|
58
|
+
self.scope = str(manifest.get("scope", "schema"))
|
|
59
|
+
|
|
60
|
+
merged_vars = self._load_vars(env, vars)
|
|
61
|
+
self.resources = self._load_resources(
|
|
62
|
+
manifest.get("resources", []) or [], merged_vars
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def _load_vars(self, env: str, overrides: dict[str, str] | None) -> dict[str, Any]:
|
|
66
|
+
vars_file = self.root_dir / "vars" / f"{env}.yml"
|
|
67
|
+
base: dict[str, Any] = {}
|
|
68
|
+
if vars_file.exists():
|
|
69
|
+
base = yaml.safe_load(vars_file.read_text()) or {}
|
|
70
|
+
if overrides:
|
|
71
|
+
base = {**base, **overrides}
|
|
72
|
+
return base
|
|
73
|
+
|
|
74
|
+
def _render(self, text: str, merged_vars: dict[str, Any]) -> str:
|
|
75
|
+
environment = Environment(undefined=StrictUndefined, autoescape=False) # nosec B701
|
|
76
|
+
try:
|
|
77
|
+
return environment.from_string(text).render(vars=merged_vars, env=self.env)
|
|
78
|
+
except UndefinedError as exc:
|
|
79
|
+
raise KeyError(f"Missing manifest variable while rendering: {exc}") from exc
|
|
80
|
+
|
|
81
|
+
def _resolve_paths(self, entry: str) -> list[Path]:
|
|
82
|
+
base = self.root_dir / "resources"
|
|
83
|
+
rel = entry if entry.endswith((".yml", ".yaml")) else f"{entry}.yml"
|
|
84
|
+
if any(ch in rel for ch in "*?["):
|
|
85
|
+
matches = sorted(base.glob(rel))
|
|
86
|
+
if not matches:
|
|
87
|
+
raise FileNotFoundError(f"No resource file matches: {base / rel}")
|
|
88
|
+
return matches
|
|
89
|
+
candidate = base / rel
|
|
90
|
+
if not candidate.exists():
|
|
91
|
+
raise FileNotFoundError(f"Resource file not found: {candidate}")
|
|
92
|
+
return [candidate]
|
|
93
|
+
|
|
94
|
+
def _load_resources(
|
|
95
|
+
self, entries: list[str], merged_vars: dict[str, Any]
|
|
96
|
+
) -> list[ResourceSpec]:
|
|
97
|
+
specs: list[ResourceSpec] = []
|
|
98
|
+
for entry in entries:
|
|
99
|
+
for path in self._resolve_paths(entry):
|
|
100
|
+
rendered = self._render(path.read_text(), merged_vars)
|
|
101
|
+
data = yaml.safe_load(rendered) or {}
|
|
102
|
+
specs.append(ResourceSpec(resource_type=path.parent.name, data=data))
|
|
103
|
+
return specs
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def from_git(cls, repo_url: str, vars: dict[str, str] | None = None) -> Manifest:
|
|
107
|
+
"""Load a manifest from a Snowflake-hosted Git repository.
|
|
108
|
+
|
|
109
|
+
Not yet implemented — local filesystem manifests only for now.
|
|
110
|
+
"""
|
|
111
|
+
raise NotImplementedError("Git-repo manifest source is not implemented yet")
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""JSON Schema generation for resource YAML files — editor autocomplete & validation.
|
|
2
|
+
|
|
3
|
+
Schemas are produced from the Pydantic resource models, which are themselves introspected from
|
|
4
|
+
``snowflake.core`` (ADR-0001). They therefore stay in sync with the SDK automatically — no hand-
|
|
5
|
+
maintained schema. Wired to the YAML language server, they give field-name autocomplete, unknown-
|
|
6
|
+
field errors (``additionalProperties: false`` from ``extra="forbid"``), type errors, and required-
|
|
7
|
+
field checks — all at authoring time, before any ``apply`` (ADR-0015).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from pinky_provider.resources.base import BaseResource
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def all_resource_classes() -> dict[str, type[BaseResource]]:
|
|
20
|
+
"""Map of resource type key → model class, across every implemented scope registry."""
|
|
21
|
+
classes: dict[str, type[BaseResource]] = {}
|
|
22
|
+
from pinky_provider.resources.account import RESOURCES as account_resources
|
|
23
|
+
|
|
24
|
+
classes.update(account_resources)
|
|
25
|
+
# database / schema registries are added here as those scopes are implemented.
|
|
26
|
+
return classes
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def resource_json_schema(resource_cls: type[BaseResource]) -> dict[str, Any]:
|
|
30
|
+
"""JSON Schema for one resource type, generated from its Pydantic model."""
|
|
31
|
+
return resource_cls.model_json_schema()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def write_schemas(out_dir: str | Path) -> list[tuple[str, Path]]:
|
|
35
|
+
"""Write one ``<type>.schema.json`` per resource type into ``out_dir``.
|
|
36
|
+
|
|
37
|
+
Returns the ``(type, path)`` pairs written, so the CLI can report them.
|
|
38
|
+
"""
|
|
39
|
+
out = Path(out_dir)
|
|
40
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
written: list[tuple[str, Path]] = []
|
|
42
|
+
for type_name, resource_cls in sorted(all_resource_classes().items()):
|
|
43
|
+
schema = resource_json_schema(resource_cls)
|
|
44
|
+
path = out / f"{type_name}.schema.json"
|
|
45
|
+
path.write_text(json.dumps(schema, indent=2) + "\n")
|
|
46
|
+
written.append((type_name, path))
|
|
47
|
+
return written
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def get_session() -> Any:
|
|
8
|
+
"""Build (or reuse) a Snowpark ``Session`` from the default Snowflake connection.
|
|
9
|
+
|
|
10
|
+
Used by the CLI in a local / CI context. Resolves credentials from the standard Snowflake
|
|
11
|
+
connection sources — ``connections.toml`` (``SNOWFLAKE_DEFAULT_CONNECTION_NAME``) or
|
|
12
|
+
``SNOWFLAKE_*`` environment variables. Never called inside a stored procedure, where the
|
|
13
|
+
session is always passed in.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
An active Snowpark ``Session``.
|
|
17
|
+
"""
|
|
18
|
+
from snowflake.snowpark import Session
|
|
19
|
+
|
|
20
|
+
return Session.builder.getOrCreate()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def build_query_tag(repo: str, schema: str, resource_type: str, name: str) -> str:
|
|
24
|
+
"""Build a structured JSON query tag for credit attribution.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
repo: Git repository name (maps to the schema being deployed).
|
|
28
|
+
schema: Target Snowflake schema (``DATABASE.SCHEMA``).
|
|
29
|
+
resource_type: Resource type being applied (e.g. ``table``, ``procedure``).
|
|
30
|
+
name: Resource object name.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
JSON string ready for :meth:`snowflake.snowpark.Session.append_query_tag`.
|
|
34
|
+
"""
|
|
35
|
+
return json.dumps(
|
|
36
|
+
{"pinky_provider": {"repo": repo, "schema": schema, resource_type: name}},
|
|
37
|
+
separators=(",", ":"),
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def inject_query_tag(
|
|
42
|
+
session: Any, repo: str, schema: str, resource_type: str, name: str
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Append a structured query tag to the active Snowflake session.
|
|
45
|
+
|
|
46
|
+
Uses :meth:`snowflake.snowpark.Session.append_query_tag` — non-destructive,
|
|
47
|
+
preserves any tag already set on the session.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
session: Active Snowpark ``Session``.
|
|
51
|
+
repo: Git repository name.
|
|
52
|
+
schema: Target Snowflake schema (``DATABASE.SCHEMA``).
|
|
53
|
+
resource_type: Resource type being applied.
|
|
54
|
+
name: Resource object name.
|
|
55
|
+
"""
|
|
56
|
+
session.append_query_tag(build_query_tag(repo, schema, resource_type, name))
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
|
|
7
|
+
from pinky_provider.resources.base import BaseResource
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Dag(BaseResource):
|
|
11
|
+
"""Meta-resource that generates a Snowflake Task DAG from a list of task descriptors.
|
|
12
|
+
|
|
13
|
+
Produces the full DAG topology: ``ROOT_TASK_WRAPPER`` (finalizer + notification)
|
|
14
|
+
and ``TASK_WRAPPER`` (per-step SP caller with SKIP cascade). Never maps 1:1 to
|
|
15
|
+
a single Snowflake object — see ``explanation/dag.md`` for the generated structure.
|
|
16
|
+
|
|
17
|
+
Attributes:
|
|
18
|
+
tasks: Ordered list of task descriptors (each maps to one ``TASK_WRAPPER``).
|
|
19
|
+
trigger: Trigger mode — ``CRON``, ``CONDITIONAL_CRON``, or ``STREAM``.
|
|
20
|
+
alerts_to: Notification target on technical failure.
|
|
21
|
+
contact_to: Notification target on business rejection.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
tasks: list[dict[str, Any]] = Field(default_factory=list)
|
|
25
|
+
trigger: str = "CRON"
|
|
26
|
+
alerts_to: str | None = None
|
|
27
|
+
contact_to: str | None = None
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pinky_provider.core.diff import ResourceDiff
|
|
6
|
+
from pinky_provider.core.manifest import Manifest
|
|
7
|
+
from pinky_provider.resources.account import AccountProvider
|
|
8
|
+
from pinky_provider.resources.database import DatabaseProvider
|
|
9
|
+
from pinky_provider.resources.schema import SchemaProvider
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Provider:
|
|
13
|
+
"""Full-scope provider — assembles schema, database, and account resources.
|
|
14
|
+
|
|
15
|
+
Entry point for CLI and CI/CD pipelines. Never use inside a stored procedure
|
|
16
|
+
(imports the full resource graph including account-level objects).
|
|
17
|
+
|
|
18
|
+
The manifest's ``scope`` selects which sub-provider runs. Only ``account`` scope is wired in
|
|
19
|
+
this first slice; ``schema`` and ``database`` follow.
|
|
20
|
+
|
|
21
|
+
Example:
|
|
22
|
+
.. code-block:: python
|
|
23
|
+
|
|
24
|
+
from pinky_provider import Provider
|
|
25
|
+
|
|
26
|
+
provider = Provider(session)
|
|
27
|
+
provider.plan(source="manifest.yml", env="SANDBOX")
|
|
28
|
+
provider.apply(source="manifest.yml", env="SANDBOX")
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
session: Active Snowpark ``Session``.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, session: Any) -> None:
|
|
35
|
+
self._session = session
|
|
36
|
+
self.schema = SchemaProvider(session)
|
|
37
|
+
self.database = DatabaseProvider(session)
|
|
38
|
+
self.account = AccountProvider(session)
|
|
39
|
+
|
|
40
|
+
def _route(self, manifest: Manifest) -> AccountProvider:
|
|
41
|
+
if manifest.scope == "account":
|
|
42
|
+
return self.account
|
|
43
|
+
raise NotImplementedError(
|
|
44
|
+
f"scope {manifest.scope!r} not implemented yet — only 'account' is wired"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def plan(
|
|
48
|
+
self,
|
|
49
|
+
source: str,
|
|
50
|
+
target: str | None = None,
|
|
51
|
+
env: str = "SANDBOX",
|
|
52
|
+
vars: dict[str, str] | None = None,
|
|
53
|
+
) -> list[ResourceDiff]:
|
|
54
|
+
"""Compute the full change plan without applying anything.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
source: Path to ``manifest.yml``.
|
|
58
|
+
target: Target schema as ``DATABASE.SCHEMA`` (schema scope) — unused at account scope.
|
|
59
|
+
env: Environment selecting ``vars/<env>.yml`` (e.g. ``SANDBOX``).
|
|
60
|
+
vars: Variable overrides injected into the manifest.
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
Ordered list of :class:`~pinky_provider.core.diff.ResourceDiff`.
|
|
64
|
+
"""
|
|
65
|
+
manifest = Manifest(source, env=env, vars=vars)
|
|
66
|
+
return self._route(manifest).plan(manifest)
|
|
67
|
+
|
|
68
|
+
def apply(
|
|
69
|
+
self,
|
|
70
|
+
source: str,
|
|
71
|
+
target: str | None = None,
|
|
72
|
+
env: str = "SANDBOX",
|
|
73
|
+
vars: dict[str, str] | None = None,
|
|
74
|
+
) -> list[ResourceDiff]:
|
|
75
|
+
"""Apply the manifest — deploys or alters all declared objects.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
source: Path to ``manifest.yml``.
|
|
79
|
+
target: Target schema as ``DATABASE.SCHEMA`` (schema scope) — unused at account scope.
|
|
80
|
+
env: Environment selecting ``vars/<env>.yml``.
|
|
81
|
+
vars: Variable overrides injected into the manifest.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
The diffs actually executed.
|
|
85
|
+
"""
|
|
86
|
+
manifest = Manifest(source, env=env, vars=vars)
|
|
87
|
+
return self._route(manifest).apply(manifest)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Scope providers, imported lazily (PEP 562).
|
|
2
|
+
|
|
3
|
+
Importing ``pinky_provider.resources`` (or the pure ``resources.base`` module) does not pull the
|
|
4
|
+
account graph — and therefore ``snowflake.core`` — until a scope provider is actually accessed.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
__all__ = ["AccountProvider", "DatabaseProvider", "SchemaProvider"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def __getattr__(name: str) -> Any:
|
|
15
|
+
if name == "AccountProvider":
|
|
16
|
+
from pinky_provider.resources.account import AccountProvider
|
|
17
|
+
|
|
18
|
+
return AccountProvider
|
|
19
|
+
if name == "DatabaseProvider":
|
|
20
|
+
from pinky_provider.resources.database import DatabaseProvider
|
|
21
|
+
|
|
22
|
+
return DatabaseProvider
|
|
23
|
+
if name == "SchemaProvider":
|
|
24
|
+
from pinky_provider.resources.schema import SchemaProvider
|
|
25
|
+
|
|
26
|
+
return SchemaProvider
|
|
27
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pinky_provider.core.diff import Action, ResourceDiff, compute_diff
|
|
6
|
+
from pinky_provider.core.manifest import Manifest, ResourceSpec
|
|
7
|
+
from pinky_provider.core.session import inject_query_tag
|
|
8
|
+
from pinky_provider.resources.account.warehouse import Warehouse
|
|
9
|
+
from pinky_provider.resources.base import BaseResource
|
|
10
|
+
|
|
11
|
+
# Resource type key → resource class. Extended as account types are implemented.
|
|
12
|
+
RESOURCES: dict[str, type[BaseResource]] = {
|
|
13
|
+
"warehouse": Warehouse,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class AccountProvider:
|
|
18
|
+
"""Account-scoped provider — deploys account-level objects (warehouses, roles, etc.).
|
|
19
|
+
|
|
20
|
+
Never imported inside a stored procedure. Used only by the CLI in CI/CD context.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
session: Active Snowpark ``Session`` with ``ACCOUNTADMIN`` or equivalent role.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, session: Any) -> None:
|
|
27
|
+
from snowflake.core import Root
|
|
28
|
+
|
|
29
|
+
self._session = session
|
|
30
|
+
self._root = Root(session)
|
|
31
|
+
|
|
32
|
+
def _build(self, spec: ResourceSpec) -> BaseResource:
|
|
33
|
+
"""Construct a validated resource model from a manifest spec.
|
|
34
|
+
|
|
35
|
+
``_``-prefixed YAML keys (``_depends_on``, ``_tags``) are stripped before validation and
|
|
36
|
+
re-attached as private attributes, so ``extra="forbid"`` only sees Snowflake fields.
|
|
37
|
+
"""
|
|
38
|
+
cls = RESOURCES.get(spec.resource_type)
|
|
39
|
+
if cls is None:
|
|
40
|
+
raise ValueError(
|
|
41
|
+
f"Unsupported account resource type: {spec.resource_type!r}"
|
|
42
|
+
)
|
|
43
|
+
meta = {k: v for k, v in spec.data.items() if k.startswith("_")}
|
|
44
|
+
fields = {k: v for k, v in spec.data.items() if not k.startswith("_")}
|
|
45
|
+
resource = cls(**fields)
|
|
46
|
+
resource._depends_on = meta.get("_depends_on", [])
|
|
47
|
+
resource._tags = meta.get("_tags", {})
|
|
48
|
+
return resource
|
|
49
|
+
|
|
50
|
+
def _fetch(self, resource: BaseResource) -> dict[str, Any] | None:
|
|
51
|
+
"""Fetch the current Snowflake state, or ``None`` if the object does not exist."""
|
|
52
|
+
from snowflake.core.exceptions import NotFoundError
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
current = resource.collection(self._root)[resource.name].fetch().to_dict()
|
|
56
|
+
except NotFoundError:
|
|
57
|
+
return None
|
|
58
|
+
return dict(current)
|
|
59
|
+
|
|
60
|
+
def _diff(self, spec: ResourceSpec, resource: BaseResource) -> ResourceDiff:
|
|
61
|
+
desired = resource.to_sdk_dict() # exactly what Snowflake will receive
|
|
62
|
+
actual = self._fetch(resource)
|
|
63
|
+
return compute_diff(
|
|
64
|
+
spec.resource_type,
|
|
65
|
+
resource.name,
|
|
66
|
+
desired,
|
|
67
|
+
actual,
|
|
68
|
+
normalizers=type(resource)._field_normalizers,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
def plan(self, manifest: Manifest) -> list[ResourceDiff]:
|
|
72
|
+
"""Compute changes without applying them."""
|
|
73
|
+
return [self._diff(spec, self._build(spec)) for spec in manifest.resources]
|
|
74
|
+
|
|
75
|
+
def apply(self, manifest: Manifest) -> list[ResourceDiff]:
|
|
76
|
+
"""Apply the manifest at account scope — creates or alters each declared object.
|
|
77
|
+
|
|
78
|
+
Returns the diff list actually executed, so the CLI can render what happened.
|
|
79
|
+
"""
|
|
80
|
+
applied: list[ResourceDiff] = []
|
|
81
|
+
for spec in manifest.resources:
|
|
82
|
+
resource = self._build(spec)
|
|
83
|
+
diff = self._diff(spec, resource)
|
|
84
|
+
if diff.action in (Action.CREATE, Action.ALTER):
|
|
85
|
+
inject_query_tag(
|
|
86
|
+
self._session,
|
|
87
|
+
repo=manifest.root_dir.name,
|
|
88
|
+
schema=manifest.scope,
|
|
89
|
+
resource_type=spec.resource_type,
|
|
90
|
+
name=resource.name,
|
|
91
|
+
)
|
|
92
|
+
resource.collection(self._root)[resource.name].create_or_alter(
|
|
93
|
+
resource.to_sdk()
|
|
94
|
+
)
|
|
95
|
+
applied.append(diff)
|
|
96
|
+
return applied
|