tundri 1.3.1__py3-none-any.whl

tundri/__init__.py

@@ -0,0 +1,3 @@
+from dotenv import load_dotenv
+
+load_dotenv()

tundri/cli.py

@@ -0,0 +1,98 @@
+import argparse
+import logging
+import sys
+
+from rich.console import Console
+from rich.logging import RichHandler
+
+from tundri.core import drop_create_objects
+from tundri.utils import (
+    run_command,
+    log_dry_run_info,
+    load_env_var
+)
+
+
+logging.basicConfig(
+    level="WARN", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
+)
+log = logging.getLogger(__name__)
+log.setLevel("INFO")
+console = Console()
+
+
+def drop_create(args):
+    console.log("[bold][purple]Drop/create Snowflake objects[/purple] started[/bold]")
+    if args.dry:
+        log_dry_run_info()
+    is_success = drop_create_objects(args.permifrost_spec_path, args.dry)
+    if is_success:
+        console.log(
+            "[bold][purple]\nDrop/create Snowflake objects[/purple] completed successfully[/bold]\n"
+        )
+    else:
+        sys.exit(1)
+
+
+def permifrost(args):
+    console.log("[bold][purple]Permifrost[/purple] started[/bold]")
+    cmd = [
+        "permifrost",
+        "run",
+        args.permifrost_spec_path,
+        "--ignore-missing-entities-dry-run",
+    ]
+
+    if args.dry:
+        cmd.append("--dry")
+        log_dry_run_info()
+
+    console.log(f"Running command: \n[italic]{' '.join(cmd)}[/italic]\n")
+    run_command(cmd)
+    console.log("[bold][purple]Permifrost[/purple] completed successfully[bold]\n")
+
+
+def run(args):
+    drop_create(args)
+    permifrost(args)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="tundri - Drop, create and alter Snowflake objects and set permissions with Permifrost"
+    )
+    subparsers = parser.add_subparsers()
+
+    # Drop/create functionality
+    parser_drop_create = subparsers.add_parser("drop_create", help="Drop, create and alter Snowflake objects")
+    parser_drop_create.add_argument(
+        "-p", "--permifrost_spec_path", "--filepath", required=True
+    )
+    parser_drop_create.add_argument("--dry", action="store_true", help="Run in dry mode")
+    parser_drop_create.set_defaults(func=drop_create)
+
+    # Permifrost functionality
+    parser_drop_create = subparsers.add_parser("permifrost", help="Run Permifrost")
+    parser_drop_create.add_argument(
+        "-p", "--permifrost_spec_path", "--filepath", required=True
+    )
+    parser_drop_create.add_argument("--dry", action="store_true", help="Run in dry mode")
+    parser_drop_create.set_defaults(func=permifrost)
+
+    # Run both
+    parser_drop_create = subparsers.add_parser("run", help="Run drop_create and then permifrost")
+    parser_drop_create.add_argument(
+        "-p", "--permifrost_spec_path", "--filepath", required=True
+    )
+    parser_drop_create.add_argument("--dry", action="store_true", help="Run in dry mode")
+    parser_drop_create.set_defaults(func=run)
+
+    args = parser.parse_args()
+    # Loading .env here, because function needs access to the path to config .yml, as
+    # the .env is expected to live in the same directory as the .yml
+    load_env_var(args.permifrost_spec_path)
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

tundri/constants.py

@@ -0,0 +1,47 @@
+import os
+
+from tundri.objects import (
+    Warehouse,
+    Database,
+    User,
+    Role,
+    Schema,
+)
+
+
+ENV_VAR_OVERRIDE_ROLE = "TUNDRI_DROP_CREATE_ROLE"
+
+OBJECT_TYPE_MAP = {
+    "warehouse": Warehouse,
+    "database": Database,
+    "user": User,
+    "role": Role,
+    "schema": Schema,
+}
+
+OBJECT_TYPES = list(OBJECT_TYPE_MAP.keys())
+
+# For backwards compatibility, we allow setting the role via environment variables
+# so previous setups that used `permifrost` role will continue to work
+OBJECT_ROLE_MAP = {
+    "warehouse": os.getenv(ENV_VAR_OVERRIDE_ROLE, "SYSADMIN"),
+    "database": os.getenv(ENV_VAR_OVERRIDE_ROLE, "SYSADMIN"),
+    "schema": os.getenv(ENV_VAR_OVERRIDE_ROLE, "SYSADMIN"),
+    "user": os.getenv(ENV_VAR_OVERRIDE_ROLE, "SECURITYADMIN"),
+    "role": os.getenv(ENV_VAR_OVERRIDE_ROLE, "SECURITYADMIN"),
+}
+# SYSADMIN can't see objects after ownership is transferred,
+# so we need to alwaysuse SECURITYADMIN for inspection
+INSPECTOR_ROLE = os.getenv(ENV_VAR_OVERRIDE_ROLE, "SECURITYADMIN")
+
+SYSTEM_DEFINED_ROLES = [
+    "accountadmin",
+    "securityadmin",
+    "sysadmin",
+    "useradmin",
+    "orgadmin",
+]
+
+STRING_CASING_CONVERSION_MAP = {
+    "rsa_public_key": str,  # Keep case
+}

tundri/core.py

@@ -0,0 +1,278 @@
+import logging
+import os
+from typing import FrozenSet, Dict, List
+
+from rich.console import Console
+from rich.logging import RichHandler
+from rich.prompt import Prompt
+from yaml import load, Loader
+
+from tundri.constants import (
+    OBJECT_TYPES,
+    OBJECT_ROLE_MAP,
+    SYSTEM_DEFINED_ROLES,
+)
+from tundri.inspector import inspect_object_type
+from tundri.objects import SnowflakeObject
+from tundri.parser import parse_object_type
+from tundri.utils import (
+    get_configs,
+    get_snowflake_cursor,
+    format_params,
+)
+
+
+all_ddl_statements = {object_type: None for object_type in OBJECT_TYPES}
+
+drop_template = "USE ROLE {role};DROP {object_type} {name};"
+create_template = "USE ROLE {role};CREATE {object_type} {name} {extra_sql};"
+alter_template = "USE ROLE {role};ALTER {object_type} {name} SET {parameters};"
+
+objects_to_ignore_in_alter = {"user": ["snowflake"]}
+params_to_ignore_in_alter = {
+    "user": ["password", "must_change_password"],
+    "warehouse": ["initially_suspended", "statement_timeout_in_seconds"],
+}
+
+
+logging.basicConfig(
+    level="WARN", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
+)
+log = logging.getLogger(__name__)
+log.setLevel("INFO")
+console = Console()
+
+# Compatible with GitHub, GitLab and Bitbucket
+IS_CI_RUN = os.getenv("CI") == "true"
+
+
+def build_statements_list(
+    statements: Dict, object_types: List[str] = OBJECT_TYPES
+) -> List:
+    """
+    Build a list of statements to be executed from a dictionary of statements with the
+    structure:
+    {
+        "user": {
+            "drop": ["DROP USER ...", "DROP USER ..."],
+            "create": ["CREATE USER ..., "CREATE USER ..."],
+            "alter": ["ALTER USER ..., "ALTER USER ..."],
+        }
+    }
+
+    To a list of statements like:
+    ["DROP USER ...", "DROP USER ...", "CREATE USER ..., "ALTER USER ..."]
+
+    Args:
+        statements: dict with the list of statements of each type (e.g. create, drop)
+                    it assumes statements come as pairs  like "USE ROLE...; CREATE/DROP ..."
+        object_types: list of object types to process, defaults to OBJECT_TYPES constant
+
+    Returns:
+        statements_seq: list with drop, create and alter statements in sequence for all
+                        object types
+    """
+    statements_seq = []
+    for object_type in object_types:
+        for operation in ["drop", "create", "alter"]:  # Order matters
+            for statement_pair in statements[object_type][operation]:
+                for s in statement_pair.split(";"):
+                    if s:  # Ignore empty strings
+                        statements_seq.append(s.strip())
+    return statements_seq
+
+
+def print_ddl_statements(statements: Dict) -> None:
+    """Print DDL statements to be executed."""
+    if not statements:
+        console.log(
+            "No statements to execute (the state of Snowflake objects matches the Permifrost spec)\n"
+        )
+        return
+    for s in statements:
+        if s.startswith("USE ROLE"):
+            continue
+        console.log(f"[italic]- {s}[/italic]")
+    console.log()
+
+
+def execute_ddl(cursor, statements: List) -> None:
+    """Execute drop, create and alter statements in sequence for each object type.
+
+    Args:
+        cursor: Snowflake API cursor object
+        statements: list with drop, create and alter statements in sequence for all
+                    object types
+    """
+    console.log("\n[bold]Executing DDL statements[/bold]:")
+    for s in statements:
+        cursor.execute(s)
+        if s.startswith("USE ROLE"):
+            continue
+        console.log(f"[green]\u2713[/green] [italic]{s}[/italic]")
+
+
+def ignore_system_defined_roles(
+    objects: FrozenSet[SnowflakeObject],
+) -> FrozenSet[SnowflakeObject]:
+    """Ignore system-defined roles to avoid errors when trying to create or drop them."""
+    return frozenset(
+        [
+            obj
+            for obj in objects
+            if not (obj.type == "role" and obj.name in SYSTEM_DEFINED_ROLES)
+        ]
+    )
+
+
+def resolve_objects(
+    existing_objects: FrozenSet[SnowflakeObject],
+    ought_objects: FrozenSet[SnowflakeObject],
+) -> Dict:
+    """Prepare DROP, CREATE and ALTER statements for an object type.
+
+    Args:
+        existing_objects: Set of Snowflake objects that currently exist
+        ought_objects: Set of Snowflake objects that are expected to exist
+
+    Returns:
+        ddl_statements: dict with drop, create and alter keys with lists of DDL statments
+                        to be executed for the given object type
+    """
+    ddl_statements = {
+        "drop": [],
+        "create": [],
+        "alter": [],
+    }
+
+    # Infer type from arguments
+    object_type = list(existing_objects)[0].type
+    console.log(f"Resolving {object_type} objects")
+
+    role = OBJECT_ROLE_MAP[object_type]
+
+    # Check which objects to drop/create/keep
+    objects_to_drop = existing_objects.difference(ought_objects)
+    if object_type == "schema":  # Schemas should not be dropped
+        objects_to_drop = frozenset()
+    objects_to_create = ought_objects.difference(existing_objects)
+    objects_to_keep = ought_objects.intersection(existing_objects)
+
+    # Remove create or drop statements for system-defined roles
+    objects_to_create = ignore_system_defined_roles(objects_to_create)
+    objects_to_drop = ignore_system_defined_roles(objects_to_drop)
+
+    # Prepare CREATE/DROP statements
+    ddl_statements["create"] = [
+        create_template.format(
+            role=role,
+            object_type=object_type,
+            name=obj.name,
+            extra_sql=format_params(obj.params),
+        ).strip()
+        for obj in objects_to_create
+    ]
+    ddl_statements["drop"] = [
+        drop_template.format(role=role, object_type=object_type, name=obj.name)
+        for obj in objects_to_drop
+    ]
+
+    # Prepare ALTER statements
+    existing_objects_to_keep = sorted(
+        [obj for obj in existing_objects if obj in objects_to_keep]
+    )
+    ought_objects_to_keep = sorted(
+        [obj for obj in ought_objects if obj in objects_to_keep]
+    )
+
+    for existing, ought in zip(existing_objects_to_keep, ought_objects_to_keep):
+        assert (
+            existing == ought
+        )  # Leverages custom __eq__ implementation to compare name and type
+        if not ought.params:
+            continue
+        if existing.params == ought.params:
+            continue
+
+        for p in params_to_ignore_in_alter.get(object_type, list()):
+            ought.params.pop(p, None)
+
+        ought_params_set = set(ought.params.items())
+        existing_params_set = set(existing.params.items())
+        params_to_alter_set = ought_params_set.difference(existing_params_set)
+        if not params_to_alter_set:
+            continue
+        if ought.name in objects_to_ignore_in_alter.get(object_type, list()):
+            continue
+        ddl_statements["alter"].append(
+            alter_template.format(
+                role=role,
+                object_type=object_type,
+                name=ought.name,
+                parameters=format_params(dict(params_to_alter_set)),
+            )
+        )
+
+    return ddl_statements
+
+
+def drop_create_objects(permifrost_spec_path: str, is_dry_run: bool):
+    """
+    Drop and create Snowflake objects based on Permifrost specification and inspection of Snowflake metadata.
+
+    Args:
+        permifrost_spec_path: path to the Permifrost specification file
+        is_dry_run: flag to run the operation in dry-run mode
+
+    Returns:
+        bool: True if the operation was successful, False otherwise
+    """
+    permifrost_spec = load(open(permifrost_spec_path, "r"), Loader=Loader)
+
+    for object_type in OBJECT_TYPES:
+        existing_objects = inspect_object_type(object_type)
+        ought_objects = parse_object_type(permifrost_spec, object_type)
+        all_ddl_statements[object_type] = resolve_objects(
+            existing_objects,
+            ought_objects,
+        )
+
+    console.log("\n[bold]DDL statements to be executed[/bold]:")
+    ddl_statements_seq = build_statements_list(all_ddl_statements)
+    print_ddl_statements(ddl_statements_seq)
+    drop_statements = [s for s in ddl_statements_seq if s.startswith("DROP")]
+
+    if IS_CI_RUN:
+        console.log(
+            "[bold][yellow]CI run detected[/bold][/yellow]: Skipping manual confirmations"
+        )
+
+    if not is_dry_run and not IS_CI_RUN:
+        configs = get_configs()
+        console.log(
+            f"\n[bold][blue]INFO[/bold][/blue]: Executing for Snowflake account: {configs['account']}"
+        )
+        user_input = Prompt.ask(
+            f"\n\t>>> Type [bold]{configs['account']}[/bold] to proceed or any other key to abort"
+        )
+        if user_input.lower() != configs["account"].lower():
+            console.log()
+            console.log("Exited without executing any statements")
+            return False
+
+    if not is_dry_run and not IS_CI_RUN and drop_statements:
+        console.log(
+            f"\n[bold][red]WARNING[/bold][/red]: The following DROP statements are about to be executed: {(drop_statements)}"
+        )
+        user_input = Prompt.ask(
+            "\n\t>>> Type [bold]drop[/bold] to proceed or any other key to abort"
+        )
+        if user_input.lower() != "drop":
+            console.log()
+            console.log("Exited without executing any statements")
+            return False
+
+    if not is_dry_run:
+        execute_ddl(get_snowflake_cursor(), ddl_statements_seq)
+
+    return True

tundri/inspector.py

@@ -0,0 +1,97 @@
+from pprint import pprint
+from typing import FrozenSet
+
+from tundri.constants import OBJECT_TYPES, OBJECT_TYPE_MAP, INSPECTOR_ROLE
+from tundri.objects import SnowflakeObject, Schema
+from tundri.utils import plural, get_snowflake_cursor, format_metadata_value
+
+# Column names of SHOW statement are different than parameter names in DDL statements
+parameter_name_map = {
+    "warehouse": {
+        "size": "warehouse_size",
+        "type": "warehouse_type",
+    },
+}
+
+
+def inspect_schemas() -> FrozenSet[Schema]:
+    """Get schemas that exist based on Snowflake metadata.
+
+    Returns:
+        inspected_objects: set of instances of `SnowflakeObject` subclasses
+    """
+    # Keys are databases and values are list of schemas e.g. {'ANALYTICS': ['REPORTING']}
+    existing_schemas = {}
+    with get_snowflake_cursor() as cursor:
+        cursor.execute(f"USE ROLE {INSPECTOR_ROLE}")
+        cursor.execute("SHOW SCHEMAS IN ACCOUNT")
+        schemas_list = [
+            (row[4], row[1]) for row in cursor
+        ]  # List of tuples: database, schema
+    for database, schema in schemas_list:
+        database = database.upper()
+        schema = schema.upper()
+        if not existing_schemas.get(database):
+            existing_schemas[database] = []
+        existing_schemas[database].append(schema)
+
+    existing_schema_names = []
+    for database, schemas in existing_schemas.items():
+        for schema in schemas:
+            existing_schema_names.append(f"{database}.{schema}")
+
+    return frozenset([Schema(name=name) for name in existing_schema_names])
+
+
+def inspect_object_type(object_type: str) -> FrozenSet[SnowflakeObject]:
+    """Initialize Snowflake objects of a given type from Snowflake metadata.
+
+    Args:
+        object_type: Object type e.g. "database", "user", etc
+
+    Returns:
+        inspected_objects: set of instances of `SnowflakeObject` subclasses
+    """
+    if object_type == "schema":
+        return inspect_schemas()
+
+    with get_snowflake_cursor() as cursor:
+        cursor.execute(f"USE ROLE {INSPECTOR_ROLE}")
+        cursor.execute(f"SHOW {plural(object_type)}")
+        desc = cursor.description
+        column_names = [
+            parameter_name_map.get(object_type, dict()).get(col[0], col[0]) for col in desc
+        ]
+        formatted_rows = [
+            tuple(
+                [
+                    format_metadata_value(column_names[idx], value)
+                    for idx, value in enumerate(row)
+                ]
+            )
+            for row in cursor
+        ]
+    data = [dict(zip(column_names, row)) for row in formatted_rows]
+
+    inspected_objects = []
+    for object in data:
+        name = object.pop("name")
+        # Ignore Snowflake system objects
+        if name.startswith("system$"):
+            continue
+        inspected_objects.append(OBJECT_TYPE_MAP[object_type](name=name, params=object))
+
+    return frozenset(inspected_objects)
+
+
+def run():
+    inspected_objects = {plural(object_type): None for object_type in OBJECT_TYPES}
+
+    inspected_objects["warehouses"] = inspect_object_type("warehouse")
+    inspected_objects["databases"] = inspect_object_type("database")
+
+    pprint(inspected_objects)
+
+
+if __name__ == "__main__":
+    run()

tundri/objects.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass, field
+from typing import Dict, Tuple
+
+
+@dataclass(frozen=True)
+class SnowflakeObject:
+    """Base class to represent Snowflake objects.
+
+    It has customized behavior for equality checks, set operations and sort. This is
+    done to allow for simpler comparisons between objects that exist vs. ought to exist.
+    Equality checks ignore the paramaters, which need to be checked using more complex logic.
+
+    Attributes:
+        type: object type, e.g. `database`, `warehouse`, etc
+        name: object name, e.g. `raw` for a database or `load` for a warehouse
+        params: dict with object parameters, e.g. for a user: {'default_warehouse': 'load'}
+        required_params: tuple with values expected as keys of `params`
+    """
+
+    type: str = None
+    name: str = None
+    params: Dict = field(default_factory=dict)
+    required_params: Tuple = field(default_factory=tuple)
+
+    def __eq__(self, other):
+        return (
+            self.type.casefold() == other.type.casefold()
+            and self.name.casefold() == other.name.casefold()
+        )
+
+    def __hash__(self):
+        return hash((self.type.casefold(), self.name.casefold()))
+
+    def __lt__(self, other):
+        return self.name.casefold() < other.name.casefold()
+
+    def get_missing_required_params(self):
+        if self.required_params and not self.params:
+            return self.required_params
+        return [key for key in self.required_params if key not in self.params.keys()]
+
+    def check_required_params(self):
+        return not self.get_missing_required_params()
+
+
+@dataclass(frozen=True, eq=False)
+class Warehouse(SnowflakeObject):
+    type: str = "warehouse"
+    required_params: Tuple = tuple(["warehouse_size", "auto_suspend"])
+
+
+@dataclass(frozen=True, eq=False)
+class Database(SnowflakeObject):
+    type: str = "database"
+
+
+@dataclass(frozen=True, eq=False)
+class Schema(SnowflakeObject):
+    type: str = "schema"
+
+
+@dataclass(frozen=True, eq=False)
+class Role(SnowflakeObject):
+    type: str = "role"
+
+
+@dataclass(frozen=True, eq=False)
+class User(SnowflakeObject):
+    type: str = "user"
+    required_params: Tuple = tuple(["default_role"])
+
+
+class ConfigurationValueError(ValueError):
+    pass

tundri/parser.py

@@ -0,0 +1,112 @@
+from yaml import load, Loader
+from pprint import pprint
+from typing import FrozenSet
+
+from tundri.constants import OBJECT_TYPES, OBJECT_TYPE_MAP
+from tundri.objects import SnowflakeObject, Schema, ConfigurationValueError
+from tundri.utils import plural, format_metadata_value
+
+PERMIFROST_YAML_FILEPATH = "examples/permifrost.yml"
+
+
+def parse_schemas(permifrost_spec: dict) -> FrozenSet[Schema]:
+    """Get schemas that ought to exist based on specific role definitions.
+
+    The way schemas are defined in Permifrost is different from the other objects. We
+    need to infer which ones need to exist based on definitions of roles that should
+    use/own them.
+
+    Args:
+        permifrost_spec: Dict with contents from Permifrost YAML file
+
+    Returns:
+        parsed_objects: set of instances of `Schema` class
+    """
+    # Keys are databases and values are list of schemas e.g. {'ANALYTICS': ['REPORTING']}
+    ought_schemas = {}
+    for role in permifrost_spec["roles"]:
+        role_name, permi_defs = list(role.items())[0]
+        if permi_defs.get("owns") and permi_defs["owns"].get("schemas"):
+            for schema in permi_defs["owns"]["schemas"]:
+                database, schema_name = schema.upper().split(".")
+                if not schema_name == "*":
+                    if not ought_schemas.get(database):
+                        ought_schemas[database] = []
+                    ought_schemas[database].append(schema_name)
+        if permi_defs.get("privileges") and permi_defs["privileges"].get("schemas"):
+            for schema in permi_defs["privileges"]["schemas"].get("read", []):
+                database, schema_name = schema.upper().split(".")
+                if not schema_name == "*":
+                    if not ought_schemas.get(database):
+                        ought_schemas[database] = []
+                    if not schema_name in ought_schemas[database]:
+                        ought_schemas[database].append(schema_name)
+            for schema in permi_defs["privileges"]["schemas"].get("write", []):
+                database, schema_name = schema.upper().split(".")
+                if not schema_name == "*":
+                    if not ought_schemas.get(database):
+                        ought_schemas[database] = []
+                    if not schema_name in ought_schemas[database]:
+                        ought_schemas[database].append(schema_name)
+
+    ought_schema_names = []
+    for database, schemas in ought_schemas.items():
+        for schema in schemas:
+            ought_schema_names.append(f"{database}.{schema}")
+
+    return frozenset([Schema(name=name) for name in ought_schema_names])
+
+
+def parse_object_type(
+    permifrost_spec: dict, object_type: str
+) -> FrozenSet[SnowflakeObject]:
+    """Initialize Snowflake objects of a given type from Permifrost spec.
+
+    Args:
+        permifrost_spec: Dict with contents from Permifrost YAML file
+        object_type: Object type e.g. "database", "user", etc
+
+    Returns:
+        parsed_objects: set of instances of `SnowflakeObject` subclasses
+    """
+    if object_type == "schema":
+        return parse_schemas(permifrost_spec)
+
+    parsed_objects = []
+    for object in permifrost_spec.get(plural(object_type), []):
+        # Each object is a dict with a single key (its name) and a dict containing the spec as value
+        object_name = list(object.keys())[0]
+        object_spec = object[object_name]
+        params = dict()
+        if "meta" in object_spec.keys():
+            params = object_spec["meta"]  # Use all contents of meta as DDL parameters
+            for name, value in params.items():
+                params[name] = format_metadata_value(name, value)
+        new_parsed_object = OBJECT_TYPE_MAP[object_type](
+            name=object_name, params=params
+        )
+        if not new_parsed_object.check_required_params():
+            raise ConfigurationValueError(
+                f"Required parameters for object '{object_name}' of type '{object_type}' missing: {new_parsed_object.get_missing_required_params()}"
+            )
+        parsed_objects.append(new_parsed_object)
+
+    return frozenset(parsed_objects)
+
+
+def run():
+    permifrost_spec = load(open(PERMIFROST_YAML_FILEPATH, "r"), Loader=Loader)
+
+    parsed_objects = {plural(object_type): None for object_type in OBJECT_TYPES}
+
+    parsed_objects["warehouses"] = parse_object_type(permifrost_spec, "warehouse")
+    parsed_objects["databases"] = parse_object_type(permifrost_spec, "database")
+    parsed_objects["roles"] = parse_object_type(permifrost_spec, "role")
+    parsed_objects["users"] = parse_object_type(permifrost_spec, "user")
+    parsed_objects["schemas"] = parse_object_type(permifrost_spec, "schema")
+
+    pprint(parsed_objects)
+
+
+if __name__ == "__main__":
+    run()

tundri/utils.py

@@ -0,0 +1,208 @@
+import logging
+import os
+import subprocess
+from typing import Dict, Type, T
+from pathlib import Path
+
+from dotenv import load_dotenv, dotenv_values
+
+from rich.console import Console
+from rich.logging import RichHandler
+from snowflake.connector import connect
+
+from tundri.constants import STRING_CASING_CONVERSION_MAP
+
+
+logging.basicConfig(
+    level="WARN", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
+)
+log = logging.getLogger(__name__)
+log.setLevel("INFO")
+console = Console()
+
+# Suppress urllib3 connection warnings from Snowflake connector
+logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
+logging.getLogger("snowflake.connector.vendored.urllib3.connectionpool").setLevel(logging.ERROR)
+
+
+class ConfigurationError(Exception):
+    pass
+
+
+def load_env_var(path_to_env: str):
+    """
+    Loads environment variables from a dotenv file.
+    Dotenv file has to live in the same directory as the Permifrost specifications file.
+    If an evironment variable with the same name already exists in on the system (e.g., 
+    in .bashrc), the existing variable is overwritten with the corresponding value from
+    the dotenv file. Filename has to be ".env".
+
+    :param path_to_env: Path to .env file
+    :return: --
+    """
+    console.log("[bold][purple]Loading environment variables [/purple] started[/bold]")
+    path_to_dotenv = (
+        Path(path_to_env)
+        .resolve()  # Converts relative to absolute path
+        .parent  # Drop filename, and only retain dir from path
+    )
+    path_to_dotenv = path_to_dotenv / ".env"
+    
+    console.log(f"Checking for [italic]{str(path_to_dotenv)}[/italic]")
+    if path_to_dotenv.is_file():
+        console.log("Found dotenv file in directory; parsing")
+        env_var = dotenv_values(path_to_dotenv)  # Dump the contents of .env in a variable
+        if not env_var:
+            console.log("Dotenv file is empty, nothing to parse")
+            console.log("Using system's environment variables instead")
+        else:
+            console.log("Loading the following environment variables from dotenv:")
+            for key, value in env_var.items():
+                console.log(f"{key}={value}")
+            console.log(
+                "\nThe following environment variables already exist on the system " + 
+                "and will be overwritten with the contents of the dotenv file:"
+            )
+            for key, value in env_var.items():
+                this_value = os.environ.get(key)
+                if this_value != None:
+                    console.log(f"{key}={this_value}")
+            load_dotenv(path_to_dotenv, override=True)
+    else:
+        console.log(f"Could not find dotenv file under {str(path_to_dotenv)}")
+        console.log("Using system's environment variables instead")
+
+
+def get_configs() -> Dict[str, str]:
+    """Get the configuration from environment variables and validate them before returning"""
+    config = {
+        "user": os.getenv("PERMISSION_BOT_USER"),
+        "password": os.getenv("PERMISSION_BOT_PASSWORD", ""),
+        "account": os.getenv("PERMISSION_BOT_ACCOUNT"),
+        "database": os.getenv("PERMISSION_BOT_DATABASE"),
+        "role": os.getenv("PERMISSION_BOT_ROLE"),
+        "warehouse": os.getenv("PERMISSION_BOT_WAREHOUSE"),
+        "key_path": os.getenv("PERMISSION_BOT_KEY_PATH"),
+        "key_passphrase": os.getenv("PERMISSION_BOT_KEY_PASSPHRASE"),
+    }
+
+    if not config["account"]:
+        raise ConfigurationError(
+            "The PERMISSION_BOT_ACCOUNT environment variable is not set"
+        )
+    if not config["database"]:
+        raise ConfigurationError(
+            "The PERMISSION_BOT_DATABASE environment variable is not set"
+        )
+    if not config["user"]:
+        raise ConfigurationError(
+            "The PERMISSION_BOT_USER environment variable is not set"
+        )
+    if not config["warehouse"]:
+        raise ConfigurationError(
+            "The PERMISSION_BOT_WAREHOUSE environment variable is not set"
+        )
+
+    return config
+
+
+def get_snowflake_cursor():
+    """Get a Snowflake cursor with support for private key authentication"""
+    config = get_configs()
+    connection_params = {
+        "user": config["user"],
+        "account": config["account"],
+        "warehouse": config["warehouse"],
+        "database": config["database"],
+    }
+
+    if config["key_path"] is not None:
+        connection_params = {
+            **connection_params,
+            "private_key_file": config["key_path"],
+            "private_key_file_pwd": config["key_passphrase"],
+        }
+    else:
+        connection_params = {
+            **connection_params,
+            "password": config["password"],
+        }
+
+    return connect(**connection_params).cursor()
+
+
+def plural(name: str) -> str:
+    return f"{name}s"
+
+
+def format_metadata_value(name: str, value):
+    """
+    Format metadata values read from the YAML file or Snowflake metadata
+
+    Most values are converted to lowercase to simplify comparisons, but other parameters
+    like `rsa_public_key` are treated differently as defined in `STRING_CASING_CONVERSION_MAP`
+    """
+    if isinstance(value, str):
+        str_callable = STRING_CASING_CONVERSION_MAP.get(name, str.lower)
+        value = str_callable(value.strip())
+        if value.casefold() == "true":
+            return True
+        if value.casefold() == "false":
+            return False
+        return value
+    return value
+
+
+def format_params(params: Dict) -> str:
+    """Returns formated list of parameters to use as arguments in DDL statements"""
+
+    def get_param_value_type(value: str) -> Type[T]:
+        if not isinstance(value, str):
+            value = str(value)
+        if value.isdigit():
+            return int
+        if value.upper() in ["TRUE", "FALSE"]:
+            return bool
+        return str
+
+    params_formatted = []
+    templates = {
+        int: "{name} = {value}",
+        bool: "{name} = {value}",
+        str: "{name} = '{value}'",
+    }
+    for name, value in params.items():
+        value_type = get_param_value_type(value)
+        # Workaround to ensure Snowflake accepts default_role, default_warehouse and
+        # default_namespace when quoted as alter statement parameters
+        if name.lower() in ["default_role", "default_warehouse", "default_namespace"]:
+            value = value.upper()
+        params_formatted.append(templates[value_type].format(name=name, value=value))
+    return ", ".join(params_formatted)
+
+
+def run_command(command):
+    process = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+    )
+
+    # Continuously read and print output
+    while True:
+        output = process.stdout.readline()
+        if output == "" and process.poll() is not None:
+            break
+        if output:
+            console.log(output.strip())
+
+    # Check for errors
+    output, errs = process.communicate()
+    if process.returncode != 0:
+        console.log(f"Error: {errs.strip()}")
+        raise subprocess.CalledProcessError(process.returncode, command, errs)
+    return output, errs
+
+
+def log_dry_run_info():
+    console.log(80 * "-")
+    console.log("[bold]Executing in [yellow]dry run mode[/yellow][/bold]")
+    console.log(80 * "-")

tundri-1.3.1.dist-info/METADATA

@@ -0,0 +1,119 @@
+Metadata-Version: 2.4
+Name: tundri
+Version: 1.3.1
+Summary: Drop, create and alter Snowflake objects and set permissions with Permifrost
+Project-URL: Homepage, https://github.com/Gemma-Analytics/tundri
+Project-URL: Repository, https://github.com/Gemma-Analytics/tundri
+Project-URL: Issues, https://github.com/Gemma-Analytics/tundri/issues
+Author-email: Gemma Analytics <bijan.soltani@gemmaanalytics.com>
+License: MIT
+Keywords: database,ddl,permifrost,snowflake
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Requires-Dist: gemma-permifrost
+Requires-Dist: python-dotenv
+Requires-Dist: pyyaml
+Requires-Dist: rich
+Requires-Dist: snowflake-connector-python
+Description-Content-Type: text/markdown
+
+<div align="center">
+  <img src="docs/images/logo.jpg" alt="Snowflake Manager Logo" width="200">
+</div>
+
+**tundri** is a Python package to declaratively create, drop, and alter Snowflake objects and manage their permissions with [Permifrost](https://gitlab.com/gitlab-data/permifrost).
+
+## Motivation
+
+Permifrost is great at managing permissions, but it doesn't create or alter objects. As [GitLab's data team handbook](https://handbook.gitlab.com/handbook/enterprise-data/platform/permifrost/) states:
+> Object creation and deletion is not managed by permifrost
+
+With only Permifrost, one would have to manually create the objects and then run Permifrost to set the permissions. This is error prone and time consuming. That is where tundri comes in.
+
+### In a nutshell
+**tundri** reads the [Permifrost spec file](https://gitlab.com/gitlab-data/permifrost#spec_file) and compares with the current state of the Snowflake account. It then creates, drops, and alters the objects to match. It leverages Permifrost's YAML `meta` tags to set attributes like `default_role` for users and `warehouse_size` for warehouses. Once the objects are created, tundri runs Permifrost to set the permissions.
+
+## Getting started
+
+### Prerequisites
+
+- Credentials to a Snowflake account with the `securityadmin` role
+- A Permifrost spec file
+
+### Install
+
+```bash
+pip install tundri
+```
+
+### Configure
+
+#### Permifrost
+Add a valid [Permifrost spec file](https://gitlab.com/gitlab-data/permifrost#spec_file) to your repository. You can use the files in the `examples` folder as reference.
+
+#### Snowflake
+Set up your Snowflake connection details in the environment variables listed below.
+
+> [!TIP]
+> You can use a `.env` file to store your credentials. Place it in the same folder as the Permifrost spec file.
+
+```bash
+PERMISSION_BOT_ACCOUNT=abc134.west-europe.azure  # Your account identifier
+PERMISSION_BOT_USER=PERMIFROST
+PERMISSION_BOT_PASSWORD=...
+PERMISSION_BOT_ROLE=SECURITYADMIN    # Permifrost requires it to be `SECURITYADMIN`
+PERMISSION_BOT_DATABASE=PERMIFROST
+PERMISSION_BOT_WAREHOUSE=ADMIN
+```
+
+### Usage
+The `run` subcommand is going to drop/create objects and run Permifrost.
+
+#### Dry run
+```bash
+tundri run --permifrost_spec_path examples/permifrost.yml --dry
+```
+
+#### Normal run
+```bash
+tundri run --permifrost_spec_path examples/permifrost.yml
+```
+
+#### Getting help
+```bash
+tundri --help
+```
+
+## Development
+### Local setup
+Install the development dependencies
+
+```bash
+uv sync
+```
+
+### Run tests
+Run the tests
+```bash
+uv run pytest -v
+```
+
+### Formatting
+Run the command below to format the code
+```bash
+uv run black .
+```
+
+### Testing locally
+Dry run with the example spec file
+```bash
+uv run tundri run --dry -p examples/permifrost.yml
+```

tundri-1.3.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+tundri/__init__.py,sha256=LG-zblOfMP6hVPKsBg0_Vu7Np90aDwZoQtZkamDOsus,46
+tundri/cli.py,sha256=sd29AnbTcUU5vDnYWid5q8yTTiiJfjo31YRXGRfrUgw,3003
+tundri/constants.py,sha256=H4CKhnJpfxQFIalTow07LElNjREmOaV2C2UuxXgrkAo,1200
+tundri/core.py,sha256=4bNENkEnqehTykF2TqmJNyLEg2EjXauUJq1A6CQLHzg,9448
+tundri/inspector.py,sha256=zhDnXxFeKLCZR06zKHkBEAxyB4sO8Do0ugdu-PN-VaM,3210
+tundri/objects.py,sha256=6nXQk-JgeDlC-ZTSwNtJbxuQ-MvFWyKnirACF1ZC-_M,2230
+tundri/parser.py,sha256=KBxKo-kOMswdYY5QoImcl14vNL6uOg-pe9Q3vUfDX8Y,4767
+tundri/utils.py,sha256=M7RpcdMD3rhHWnO_GTNBd0jFwJmhos0f2FRXHTINg7I,7235
+tundri-1.3.1.dist-info/METADATA,sha256=zDTIBCpyQlXRotAQZXmcp6lHDGl3ToYFOOf0dUimJXE,3853
+tundri-1.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+tundri-1.3.1.dist-info/entry_points.txt,sha256=OyOLF3YkcU4ah14hFwSxSJbhbwMnBLDVE8ymaPbfoYI,43
+tundri-1.3.1.dist-info/RECORD,,

tundri-1.3.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

tundri-1.3.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tundri = tundri.cli:main