tundri 1.3.2__py3-none-any.whl → 1.3.3__py3-none-any.whl

tundri/cli.py

@@ -25,7 +25,9 @@ def drop_create(args):
     console.log("[bold][purple]Drop/create Snowflake objects[/purple] started[/bold]")
     if args.dry:
         log_dry_run_info()
-    is_success = drop_create_objects(args.permifrost_spec_path, args.dry)
+    is_success = drop_create_objects(
+        args.permifrost_spec_path, args.dry, args.users_to_skip
+    )
     if is_success:
         console.log(
             "[bold][purple]\nDrop/create Snowflake objects[/purple] completed successfully[/bold]\n"
@@ -62,6 +64,13 @@ def main():
         description="tundri - Drop, create and alter Snowflake objects and set permissions with Permifrost"
     )
     subparsers = parser.add_subparsers()
+    help_str_users_to_skip = """
+        Users to ignore from drop, create, and alter operations (space-separated list, case-sensitive).
+        Users with admin priviliges can't be inspected by the permifrost user, because
+        of them being higher in the role hierarchy then the default tundri inspector 
+        role. To avoid permission errors, skip those users during object inspection.
+        Altering skipped users through tundri won't work and needs to be done manually! 
+    """
 
     # Drop/create functionality
     parser_drop_create = subparsers.add_parser("drop_create", help="Drop, create and alter Snowflake objects")
@@ -69,6 +78,13 @@ def main():
         "-p", "--permifrost_spec_path", "--filepath", required=True
     )
     parser_drop_create.add_argument("--dry", action="store_true", help="Run in dry mode")
+    parser_drop_create.add_argument(
+        "--users-to-skip",
+        nargs="+",
+        metavar="USER_NAME",
+        default=["admin", "snowflake", "auto_dba"],
+        help=help_str_users_to_skip,
+    )
     parser_drop_create.set_defaults(func=drop_create)
 
     # Permifrost functionality
@@ -85,6 +101,13 @@ def main():
         "-p", "--permifrost_spec_path", "--filepath", required=True
     )
     parser_drop_create.add_argument("--dry", action="store_true", help="Run in dry mode")
+    parser_drop_create.add_argument(
+        "--users-to-skip",
+        nargs="+",
+        metavar="USER_NAME",
+        default=["admin", "snowflake", "auto_dba"],
+        help=help_str_users_to_skip,
+    )
     parser_drop_create.set_defaults(func=run)
 
     args = parser.parse_args()

tundri/core.py

@@ -19,6 +19,7 @@ from tundri.utils import (
     get_configs,
     get_snowflake_cursor,
     format_params,
+    get_existing_user,
 )
 
 
@@ -96,20 +97,20 @@ def print_ddl_statements(statements: Dict) -> None:
     console.log()
 
 
-def execute_ddl(cursor, statements: List) -> None:
+def execute_ddl(statements: List) -> None:
     """Execute drop, create and alter statements in sequence for each object type.
 
     Args:
-        cursor: Snowflake API cursor object
         statements: list with drop, create and alter statements in sequence for all
                     object types
     """
     console.log("\n[bold]Executing DDL statements[/bold]:")
-    for s in statements:
-        cursor.execute(s)
-        if s.startswith("USE ROLE"):
-            continue
-        console.log(f"[green]\u2713[/green] [italic]{s}[/italic]")
+    with get_snowflake_cursor() as cursor:
+        for s in statements:
+            cursor.execute(s)
+            if s.startswith("USE ROLE"):
+                continue
+            console.log(f"[green]\u2713[/green] [italic]{s}[/italic]")
 
 
 def ignore_system_defined_roles(
@@ -125,6 +126,24 @@ def ignore_system_defined_roles(
     )
 
 
+def ignore_existing_users(
+    objects: FrozenSet[SnowflakeObject],
+) -> FrozenSet[SnowflakeObject]:
+    """
+    Ignore users that already exist
+
+    Args:
+        cursor: Active Snowflake cursor
+        ought_objects: User objects to check
+
+    Returns:
+        "objects" parameter, but pruned of existing users
+    """
+    with get_snowflake_cursor() as cursor:
+        users_list = get_existing_user(cursor)  # List of users (Strings)
+        return frozenset([obj for obj in objects if obj.name.lower() not in users_list])
+
+
 def resolve_objects(
     existing_objects: FrozenSet[SnowflakeObject],
     ought_objects: FrozenSet[SnowflakeObject],
@@ -161,6 +180,13 @@ def resolve_objects(
     # Remove create or drop statements for system-defined roles
     objects_to_create = ignore_system_defined_roles(objects_to_create)
     objects_to_drop = ignore_system_defined_roles(objects_to_drop)
+    if object_type == "user":
+        # Since we are skipping some users with admin priviliges during object inspection,
+        # tundri won't know whether those users already exist, and will try to create them
+        # even if they already exist. Adding a IF NOT EXIST flag to the CREATE command
+        # will only work partially, because tundri still would issue prompts for the
+        # affected users
+        objects_to_create = ignore_existing_users(objects_to_create)
 
     # Prepare CREATE/DROP statements
     ddl_statements["create"] = [
@@ -216,13 +242,16 @@ def resolve_objects(
     return ddl_statements
 
 
-def drop_create_objects(permifrost_spec_path: str, is_dry_run: bool):
+def drop_create_objects(
+    permifrost_spec_path: str, is_dry_run: bool, users_to_skip: List[str]
+):
     """
     Drop and create Snowflake objects based on Permifrost specification and inspection of Snowflake metadata.
 
     Args:
         permifrost_spec_path: path to the Permifrost specification file
         is_dry_run: flag to run the operation in dry-run mode
+        users_to_skip: list of users to skip during rom drop, create, alter operations
 
     Returns:
         bool: True if the operation was successful, False otherwise
@@ -230,7 +259,7 @@ def drop_create_objects(permifrost_spec_path: str, is_dry_run: bool):
     permifrost_spec = load(open(permifrost_spec_path, "r"), Loader=Loader)
 
     for object_type in OBJECT_TYPES:
-        existing_objects = inspect_object_type(object_type)
+        existing_objects = inspect_object_type(object_type, users_to_skip)
         ought_objects = parse_object_type(permifrost_spec, object_type)
         all_ddl_statements[object_type] = resolve_objects(
             existing_objects,
@@ -273,6 +302,6 @@ def drop_create_objects(permifrost_spec_path: str, is_dry_run: bool):
             return False
 
     if not is_dry_run:
-        execute_ddl(get_snowflake_cursor(), ddl_statements_seq)
+        execute_ddl(ddl_statements_seq)
 
     return True

tundri/inspector.py

@@ -1,9 +1,20 @@
+import logging
 from pprint import pprint
-from typing import FrozenSet
+from typing import FrozenSet, List
+
+from rich.console import Console
+from rich.logging import RichHandler
 
 from tundri.constants import OBJECT_TYPES, OBJECT_TYPE_MAP, INSPECTOR_ROLE
-from tundri.objects import SnowflakeObject, Schema
-from tundri.utils import plural, get_snowflake_cursor, format_metadata_value
+from tundri.objects import SnowflakeObject, Schema, User
+from tundri.utils import (
+    plural,
+    get_snowflake_cursor,
+    format_metadata_value,
+    get_existing_user,
+)
+
+from snowflake.connector.errors import ProgrammingError
 
 # Column names of SHOW statement are different than parameter names in DDL statements
 parameter_name_map = {
@@ -14,6 +25,14 @@ parameter_name_map = {
 }
 
 
+logging.basicConfig(
+    level="WARN", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
+)
+log = logging.getLogger(__name__)
+log.setLevel("INFO")
+console = Console()
+
+
 def inspect_schemas() -> FrozenSet[Schema]:
     """Get schemas that exist based on Snowflake metadata.
 
@@ -43,17 +62,75 @@ def inspect_schemas() -> FrozenSet[Schema]:
     return frozenset([Schema(name=name) for name in existing_schema_names])
 
 
-def inspect_object_type(object_type: str) -> FrozenSet[SnowflakeObject]:
+def inspect_users(users_to_skip: List[str]) -> FrozenSet[User]:
+    """
+    Get metadata of USER objects, using Snowflake's DESCRIBE command.
+
+    Note:
+    We are using Snowflake's SHOW instead of it's DESCRIBE command to inspect
+    objects. For most objects (Databases, Schemas, Warehouses), SHOW allows us to
+    fetch object metadata, while DESCRIBE would only the structure/schema of
+    a single object.
+
+    The only exception to this are USER objects: those objects have no internal
+    structure, and their metadata essentially describes their structure. Thus,
+    SHOW and DESCRIBE return similar information, with DESCRIBE returning a more
+    complete set of metadata of a user. The following attributes are missing from
+    DESCRIBE and need to be added manually if required:
+    [created_on, owner, last_success_login, expires_at_time, locked_until_time,
+    has_password, has_rsa_public_key]
+
+    Returns:
+        data: Immutable set of user objects
+    """
+    data = []
+    with get_snowflake_cursor() as cursor:
+        users_list = get_existing_user(cursor)  # List of users (Strings)
+        for user in users_list:
+            try:
+                cursor.execute(f"USE ROLE {INSPECTOR_ROLE}")
+                cursor.execute(f"DESCRIBE USER {user}")
+
+                # DESCRIBE returns one row per user attribute, while SHOW returns one column
+                # per user attribute. Pivot the result of DESCRIBE so it works with the
+                # `format_metadata_value()` function
+                attributes = {
+                    row[0]: row[1] for row in cursor
+                }  # Dict of user attributes in the form "attribute: value"
+                formatted_row = {
+                    key.lower(): format_metadata_value(key.lower(), value)
+                    for _, (key, value) in enumerate(attributes.items())
+                }  # `format_metadata_value()` expects keys to be lowercase
+                name = formatted_row.pop("name")
+                data.append(User(name=name, params=formatted_row))
+            except ProgrammingError as e:
+                if "insufficient privileges" in e.msg.lower() and user in users_to_skip:
+                    console.log(
+                        "[bold][red]WARNING[/bold][/red]: Skipping metadata retrieval",
+                        f"for user {user}: Permifrost user doesn't have DESCRIBE",
+                        "privileges on this object",
+                    )
+                else:
+                    raise e
+    return frozenset(data)
+
+
+def inspect_object_type(
+    object_type: str, users_to_skip: List[str]
+) -> FrozenSet[SnowflakeObject]:
     """Initialize Snowflake objects of a given type from Snowflake metadata.
 
     Args:
         object_type: Object type e.g. "database", "user", etc
+        users_to_skip: list of users to skip during inspection
 
     Returns:
         inspected_objects: set of instances of `SnowflakeObject` subclasses
     """
     if object_type == "schema":
         return inspect_schemas()
+    if object_type == "user":
+        return inspect_users(users_to_skip)
 
     with get_snowflake_cursor() as cursor:
         cursor.execute(f"USE ROLE {INSPECTOR_ROLE}")

tundri/utils.py

@@ -1,7 +1,7 @@
 import logging
 import os
 import subprocess
-from typing import Dict, Type, T
+from typing import Dict, Type, T, List
 from pathlib import Path
 
 from dotenv import load_dotenv, dotenv_values
@@ -9,8 +9,9 @@ from dotenv import load_dotenv, dotenv_values
 from rich.console import Console
 from rich.logging import RichHandler
 from snowflake.connector import connect
+from snowflake.connector.cursor import SnowflakeCursor
 
-from tundri.constants import STRING_CASING_CONVERSION_MAP
+from tundri.constants import STRING_CASING_CONVERSION_MAP, INSPECTOR_ROLE
 
 
 logging.basicConfig(
@@ -206,3 +207,18 @@ def log_dry_run_info():
     console.log(80 * "-")
     console.log("[bold]Executing in [yellow]dry run mode[/yellow][/bold]")
     console.log(80 * "-")
+
+
+def get_existing_user(cursor: SnowflakeCursor) -> List[str]:
+    """
+    Fetch a list of existing usernames from Snowflake
+
+    Args:
+        cursor: active Snowflake cursor
+
+    Returns:
+        List of names from existing Snowflake user
+    """
+    cursor.execute(f"USE ROLE {INSPECTOR_ROLE}")
+    cursor.execute(f"SHOW USERS")
+    return [row[0].lower() for row in cursor]  # List of user names (Strings)

{tundri-1.3.2.dist-info → tundri-1.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tundri
-Version: 1.3.2
+Version: 1.3.3
 Summary: Drop, create and alter Snowflake objects and set permissions with Permifrost
 Project-URL: Homepage, https://github.com/Gemma-Analytics/tundri
 Project-URL: Repository, https://github.com/Gemma-Analytics/tundri
@@ -45,7 +45,7 @@ With only Permifrost, one would have to manually create the objects and then run
 
 ### Prerequisites
 
-- Credentials to a Snowflake account with the `securityadmin` role
+- Credentials to a Snowflake user account with the `securityadmin` role
 - A Permifrost spec file
 
 ### Install
@@ -117,3 +117,31 @@ Dry run with the example spec file
 ```bash
 uv run tundri run --dry -p examples/permifrost.yml
 ```
+
+## Contributing
+
+### Release process
+
+The release process is automated using GitHub Actions. Here's how it works:
+
+1. **Adding new features or bug fixes**
+   - PR tests run automatically to verify the changes on each PR
+   - Multiple PRs can be merged to main until a release-ready state is reached
+
+1. **Initiating a Release**
+   - A maintainer triggers the manual release workflow
+   - They specify the version bump type (`major`, `minor`, or `patch`)
+   - This creates a release branch and PR with updated version
+
+1. **Release Creation**
+   - When the release PR is merged to main:
+     - A Git tag is created (e.g., `v1.2.3`)
+     - A GitHub release is created
+     - The package is published to PyPI
+
+The process requires the following GitHub secrets to be configured:
+- `PYPI_API_TOKEN`: For production PyPI publishing
+- `TEST_PYPI_API_TOKEN`: For TestPyPI publishing
+- `SNOWFLAKE_*`: Snowflake credentials for running tests
+
+For full details on the release workflow, see [RELEASE_WORKFLOW.md](docs/RELEASE_WORKFLOW.md).

tundri-1.3.3.dist-info/RECORD

@@ -0,0 +1,12 @@
+tundri/__init__.py,sha256=LG-zblOfMP6hVPKsBg0_Vu7Np90aDwZoQtZkamDOsus,46
+tundri/cli.py,sha256=QWXZSkqrTIxRaG3k_y06f5FgnTxW-1hikPX2wFl4nQ0,3943
+tundri/constants.py,sha256=H4CKhnJpfxQFIalTow07LElNjREmOaV2C2UuxXgrkAo,1200
+tundri/core.py,sha256=8hlhdtU3hbAYFbg2qkthfww8MpEzGNKslcQBJuALNA0,10599
+tundri/inspector.py,sha256=bA5Vl6D8pe0k5rLSVY1w-ms8e0ZWYv8VQ2G9yJuvxZw,6284
+tundri/objects.py,sha256=6nXQk-JgeDlC-ZTSwNtJbxuQ-MvFWyKnirACF1ZC-_M,2230
+tundri/parser.py,sha256=KBxKo-kOMswdYY5QoImcl14vNL6uOg-pe9Q3vUfDX8Y,4767
+tundri/utils.py,sha256=-PHp4iw2GIrtnIqaw3SS5Eyu0DDT4GXnQoC-afGcfSA,7722
+tundri-1.3.3.dist-info/METADATA,sha256=_vnargta2DAprJGnOwbIMjYc94bgHYzgmVcL890jKo0,4876
+tundri-1.3.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+tundri-1.3.3.dist-info/entry_points.txt,sha256=OyOLF3YkcU4ah14hFwSxSJbhbwMnBLDVE8ymaPbfoYI,43
+tundri-1.3.3.dist-info/RECORD,,

tundri-1.3.2.dist-info/RECORD

@@ -1,12 +0,0 @@
-tundri/__init__.py,sha256=LG-zblOfMP6hVPKsBg0_Vu7Np90aDwZoQtZkamDOsus,46
-tundri/cli.py,sha256=sd29AnbTcUU5vDnYWid5q8yTTiiJfjo31YRXGRfrUgw,3003
-tundri/constants.py,sha256=H4CKhnJpfxQFIalTow07LElNjREmOaV2C2UuxXgrkAo,1200
-tundri/core.py,sha256=4bNENkEnqehTykF2TqmJNyLEg2EjXauUJq1A6CQLHzg,9448
-tundri/inspector.py,sha256=zhDnXxFeKLCZR06zKHkBEAxyB4sO8Do0ugdu-PN-VaM,3210
-tundri/objects.py,sha256=6nXQk-JgeDlC-ZTSwNtJbxuQ-MvFWyKnirACF1ZC-_M,2230
-tundri/parser.py,sha256=KBxKo-kOMswdYY5QoImcl14vNL6uOg-pe9Q3vUfDX8Y,4767
-tundri/utils.py,sha256=M7RpcdMD3rhHWnO_GTNBd0jFwJmhos0f2FRXHTINg7I,7235
-tundri-1.3.2.dist-info/METADATA,sha256=xhJU41VP3M9wsLWoSvtDvKMbRhVnJsbUauxT_L1CK8w,3842
-tundri-1.3.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-tundri-1.3.2.dist-info/entry_points.txt,sha256=OyOLF3YkcU4ah14hFwSxSJbhbwMnBLDVE8ymaPbfoYI,43
-tundri-1.3.2.dist-info/RECORD,,