meltano-state-backend-snowflake 0.1.0__py3-none-any.whl

meltano_state_backend_snowflake/__init__.py

@@ -0,0 +1 @@
+"""Meltano State Backend for Snowflake."""

meltano_state_backend_snowflake/backend.py

@@ -0,0 +1,410 @@
+"""StateStoreManager for Snowflake state backend."""
+
+from __future__ import annotations
+
+import json
+import typing as t
+from contextlib import contextmanager
+from functools import cached_property
+from time import sleep
+from urllib.parse import urlparse
+
+import snowflake.connector
+from meltano.core.error import MeltanoError
+from meltano.core.setting_definition import SettingDefinition, SettingKind
+from meltano.core.state_store.base import (
+    MeltanoState,
+    MissingStateBackendSettingsError,
+    StateIDLockedError,
+    StateStoreManager,
+)
+
+if t.TYPE_CHECKING:
+    from collections.abc import Generator, Iterable
+
+
+class SnowflakeStateBackendError(MeltanoError):
+    """Base error for Snowflake state backend."""
+
+
+SNOWFLAKE_ACCOUNT = SettingDefinition(
+    name="state_backend.snowflake.account",
+    label="Snowflake Account",
+    description="Snowflake account identifier",
+    kind=SettingKind.STRING,
+    env_specific=True,
+)
+
+SNOWFLAKE_USER = SettingDefinition(
+    name="state_backend.snowflake.user",
+    label="Snowflake User",
+    description="Snowflake username",
+    kind=SettingKind.STRING,
+    env_specific=True,
+)
+
+SNOWFLAKE_PASSWORD = SettingDefinition(
+    name="state_backend.snowflake.password",
+    label="Snowflake Password",
+    description="Snowflake password",
+    kind=SettingKind.STRING,
+    sensitive=True,
+    env_specific=True,
+)
+
+SNOWFLAKE_WAREHOUSE = SettingDefinition(
+    name="state_backend.snowflake.warehouse",
+    label="Snowflake Warehouse",
+    description="Snowflake compute warehouse",
+    kind=SettingKind.STRING,
+    env_specific=True,
+)
+
+SNOWFLAKE_DATABASE = SettingDefinition(
+    name="state_backend.snowflake.database",
+    label="Snowflake Database",
+    description="Snowflake database name",
+    kind=SettingKind.STRING,
+    env_specific=True,
+)
+
+SNOWFLAKE_SCHEMA = SettingDefinition(
+    name="state_backend.snowflake.schema",
+    label="Snowflake Schema",
+    description="Snowflake schema name",
+    kind=SettingKind.STRING,
+    default="PUBLIC",
+    env_specific=True,
+)
+
+SNOWFLAKE_ROLE = SettingDefinition(
+    name="state_backend.snowflake.role",
+    label="Snowflake Role",
+    description="Snowflake role to use",
+    kind=SettingKind.STRING,
+    env_specific=True,
+)
+
+
+class SnowflakeStateStoreManager(StateStoreManager):
+    """State backend for Snowflake."""
+
+    label: str = "Snowflake"
+    table_name: str = "meltano_state"
+    lock_table_name: str = "meltano_state_locks"
+
+    def __init__(
+        self,
+        uri: str,
+        *,
+        account: str | None = None,
+        user: str | None = None,
+        password: str | None = None,
+        warehouse: str | None = None,
+        database: str | None = None,
+        schema: str | None = None,
+        role: str | None = None,
+        **kwargs: t.Any,
+    ) -> None:
+        """Initialize the SnowflakeStateStoreManager.
+
+        Args:
+            uri: The state backend URI
+            account: Snowflake account identifier
+            user: Snowflake username
+            password: Snowflake password
+            warehouse: Snowflake compute warehouse
+            database: Snowflake database name
+            schema: Snowflake schema name (default: PUBLIC)
+            role: Optional Snowflake role to use
+            kwargs: Additional keyword args to pass to parent
+
+        """
+        super().__init__(**kwargs)
+        self.uri = uri
+        parsed = urlparse(uri)
+
+        # Extract connection details from URI and parameters
+        self.account = account or parsed.hostname
+        if not self.account:
+            msg = "Snowflake account is required"
+            raise MissingStateBackendSettingsError(msg)
+
+        self.user = user or parsed.username
+        if not self.user:
+            msg = "Snowflake user is required"
+            raise MissingStateBackendSettingsError(msg)
+
+        self.password = password or parsed.password
+        if not self.password:
+            msg = "Snowflake password is required"
+            raise MissingStateBackendSettingsError(msg)
+
+        self.warehouse = warehouse
+        if not self.warehouse:
+            msg = "Snowflake warehouse is required"
+            raise MissingStateBackendSettingsError(msg)
+
+        # Extract database from path
+        path_parts = parsed.path.strip("/").split("/") if parsed.path else []
+        self.database = database or (path_parts[0] if path_parts else None)
+        if not self.database:
+            msg = "Snowflake database is required"
+            raise MissingStateBackendSettingsError(msg)
+
+        self.schema = schema or (path_parts[1] if len(path_parts) > 1 else "PUBLIC")
+        self.role = role
+
+        self._ensure_tables()
+
+    @cached_property
+    def connection(self) -> snowflake.connector.SnowflakeConnection:
+        """Get a Snowflake connection.
+
+        Returns:
+            A Snowflake connection object.
+
+        """
+        conn_params = {
+            "account": self.account,
+            "user": self.user,
+            "password": self.password,
+            "warehouse": self.warehouse,
+            "database": self.database,
+            "schema": self.schema,
+        }
+        if self.role:
+            conn_params["role"] = self.role
+
+        return snowflake.connector.connect(**conn_params)
+
+    def _ensure_tables(self) -> None:
+        """Ensure the state and lock tables exist."""
+        with self.connection.cursor() as cursor:
+            # Create state table
+            cursor.execute(
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.database}.{self.schema}.{self.table_name} (
+                    state_id VARCHAR PRIMARY KEY,
+                    partial_state VARIANT,
+                    completed_state VARIANT,
+                    updated_at TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP()
+                )
+                """,
+            )
+
+            # Create lock table
+            cursor.execute(
+                f"""
+                CREATE TABLE IF NOT EXISTS {self.database}.{self.schema}.{self.lock_table_name} (
+                    state_id VARCHAR PRIMARY KEY,
+                    locked_at TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP(),
+                    lock_id VARCHAR
+                )
+                """,
+            )
+
+    def set(self, state: MeltanoState) -> None:
+        """Set the job state for the given state_id.
+
+        Args:
+            state: the state to set.
+
+        """
+        partial_json = json.dumps(state.partial_state) if state.partial_state else None
+        completed_json = json.dumps(state.completed_state) if state.completed_state else None
+
+        with self.connection.cursor() as cursor:
+            cursor.execute(
+                f"""
+                MERGE INTO {self.database}.{self.schema}.{self.table_name} AS target
+                USING (SELECT %s AS state_id, PARSE_JSON(%s) AS partial_state,
+                       PARSE_JSON(%s) AS completed_state) AS source
+                ON target.state_id = source.state_id
+                WHEN MATCHED THEN
+                    UPDATE SET
+                        partial_state = source.partial_state,
+                        completed_state = source.completed_state,
+                        updated_at = CURRENT_TIMESTAMP()
+                WHEN NOT MATCHED THEN
+                    INSERT (state_id, partial_state, completed_state)
+                    VALUES (source.state_id, source.partial_state, source.completed_state)
+                """,  # noqa: S608
+                (state.state_id, partial_json, completed_json),
+            )
+
+    def get(self, state_id: str) -> MeltanoState | None:
+        """Get the job state for the given state_id.
+
+        Args:
+            state_id: the name of the job to get state for.
+
+        Returns:
+            The current state for the given job
+
+        """
+        with self.connection.cursor() as cursor:
+            cursor.execute(
+                f"""
+                SELECT partial_state, completed_state
+                FROM {self.database}.{self.schema}.{self.table_name}
+                WHERE state_id = %s
+                """,  # noqa: S608
+                (state_id,),
+            )
+            row = cursor.fetchone()
+
+            if not row:
+                return None
+
+            # Snowflake returns None for NULL VARIANT columns
+            # but MeltanoState expects empty dicts
+            # Additionally, VARIANT columns might return JSON strings that need parsing
+            partial_state = row[0]
+            completed_state = row[1]
+
+            # Handle None values
+            if partial_state is None:
+                partial_state = {}
+            # Parse JSON string if Snowflake returns string instead of dict
+            elif isinstance(partial_state, str):
+                partial_state = json.loads(partial_state)
+
+            if completed_state is None:
+                completed_state = {}
+            # Parse JSON string if Snowflake returns string instead of dict
+            elif isinstance(completed_state, str):
+                completed_state = json.loads(completed_state)
+
+            return MeltanoState(
+                state_id=state_id,
+                partial_state=partial_state,
+                completed_state=completed_state,
+            )
+
+    def delete(self, state_id: str) -> None:
+        """Delete state for the given state_id.
+
+        Args:
+            state_id: the state_id to clear state for
+
+        """
+        with self.connection.cursor() as cursor:
+            cursor.execute(
+                f"DELETE FROM {self.database}.{self.schema}.{self.table_name} WHERE state_id = %s",  # noqa: S608
+                (state_id,),
+            )
+
+    def clear_all(self) -> int:
+        """Clear all states.
+
+        Returns:
+            The number of states cleared from the store.
+
+        """
+        with self.connection.cursor() as cursor:
+            cursor.execute(
+                f"SELECT COUNT(*) FROM {self.database}.{self.schema}.{self.table_name}",  # noqa: S608
+            )
+            count = cursor.fetchone()[0]  # type: ignore[index]
+            cursor.execute(
+                f"TRUNCATE TABLE {self.database}.{self.schema}.{self.table_name}",
+            )
+            return count  # type: ignore[no-any-return]
+
+    def get_state_ids(self, pattern: str | None = None) -> Iterable[str]:
+        """Get all state_ids available in this state store manager.
+
+        Args:
+            pattern: glob-style pattern to filter by
+
+        Returns:
+            An iterable of state_ids
+
+        """
+        with self.connection.cursor() as cursor:
+            if pattern and pattern != "*":
+                # Convert glob pattern to SQL LIKE pattern
+                sql_pattern = pattern.replace("*", "%").replace("?", "_")
+                cursor.execute(
+                    f"SELECT state_id FROM {self.database}.{self.schema}.{self.table_name} WHERE state_id LIKE %s",  # noqa: E501, S608
+                    (sql_pattern,),
+                )
+            else:
+                cursor.execute(
+                    f"SELECT state_id FROM {self.database}.{self.schema}.{self.table_name}",  # noqa: S608
+                )
+
+            for row in cursor:
+                yield row[0]
+
+    @contextmanager
+    def acquire_lock(
+        self,
+        state_id: str,
+        *,
+        retry_seconds: int = 1,
+    ) -> Generator[None, None, None]:
+        """Acquire a lock for the given job's state.
+
+        Args:
+            state_id: the state_id to lock
+            retry_seconds: the number of seconds to wait before retrying
+
+        Yields:
+            None
+
+        Raises:
+            StateIDLockedError: if the lock cannot be acquired
+
+        """
+        import uuid
+
+        lock_id = str(uuid.uuid4())
+        max_seconds = 30
+        seconds_waited = 0
+
+        while seconds_waited < max_seconds:  # pragma: no branch
+            try:
+                with self.connection.cursor() as cursor:
+                    # Try to acquire lock
+                    cursor.execute(
+                        f"""
+                        INSERT INTO {self.database}.{self.schema}.{self.lock_table_name} (state_id, lock_id)
+                        VALUES (%s, %s)
+                        """,  # noqa: E501, S608
+                        (state_id, lock_id),
+                    )
+                    break
+            except snowflake.connector.errors.ProgrammingError as e:
+                # Check if it's a unique constraint violation
+                if "Duplicate key" in str(e):
+                    seconds_waited += retry_seconds
+                    if seconds_waited >= max_seconds:  # Last attempt
+                        msg = f"Could not acquire lock for state_id: {state_id}"
+                        raise StateIDLockedError(msg) from e
+                    sleep(retry_seconds)
+                else:
+                    raise
+
+        try:
+            yield
+        finally:
+            # Release the lock
+            with self.connection.cursor() as cursor:
+                cursor.execute(
+                    f"""
+                    DELETE FROM {self.database}.{self.schema}.{self.lock_table_name}
+                    WHERE state_id = %s AND lock_id = %s
+                    """,  # noqa: S608
+                    (state_id, lock_id),
+                )
+
+            # Clean up old locks (older than 5 minutes)
+            with self.connection.cursor() as cursor:
+                cursor.execute(
+                    f"""
+                    DELETE FROM {self.database}.{self.schema}.{self.lock_table_name}
+                    WHERE locked_at < DATEADD(minute, -5, CURRENT_TIMESTAMP())
+                    """,  # noqa: S608
+                )

meltano_state_backend_snowflake-0.1.0.dist-info/METADATA

@@ -0,0 +1,129 @@
+Metadata-Version: 2.4
+Name: meltano-state-backend-snowflake
+Version: 0.1.0
+Summary: Meltano State Backend for Snowflake
+Author-email: Taylor Murphy <taylor@arch.dev>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.9
+Requires-Dist: meltano>=3.7
+Requires-Dist: snowflake-connector-python<4,>=3
+Description-Content-Type: text/markdown
+
+# `meltano-state-backend-snowflake`
+
+[![PyPI version](https://img.shields.io/pypi/v/meltano-state-backend-snowflake.svg?logo=pypi&logoColor=FFE873&color=blue)](https://pypi.org/project/meltano-state-backend-snowflake)
+[![Python versions](https://img.shields.io/pypi/pyversions/meltano-state-backend-snowflake.svg?logo=python&logoColor=FFE873)](https://pypi.org/project/meltano-state-backend-snowflake)
+
+This is a [Meltano][meltano] extension that provides a [Snowflake][snowflake] [state backend][state-backend].
+
+## Installation
+
+This package needs to be installed in the same Python environment as Meltano.
+
+### From GitHub
+
+#### With [uv]
+
+```bash
+uv tool install --with meltano-state-backend-snowflake meltano
+```
+
+#### With [pipx]
+
+```bash
+pipx install meltano
+pipx inject meltano 'meltano-state-backend-snowflake
+```
+
+## Configuration
+
+To store state in Snowflake, set the `state_backend.uri` setting to `snowflake://<user>:<password>@<account>/<database>/<schema>`.
+
+State will be stored in two tables that Meltano will create automatically:
+- `meltano_state` - Stores the actual state data
+- `meltano_state_locks` - Manages concurrency locks
+
+To authenticate to Snowflake, you'll need to provide:
+
+```yaml
+state_backend:
+  uri: snowflake://my_user:my_password@my_account/my_database/my_schema
+  snowflake:
+    warehouse: my_warehouse  # Required: The compute warehouse to use
+    role: my_role           # Optional: The role to use for the connection
+```
+
+Alternatively, you can provide credentials via individual settings:
+
+```yaml
+state_backend:
+  uri: snowflake://my_account/my_database/my_schema
+  snowflake:
+    account: my_account
+    user: my_user
+    password: my_password
+    warehouse: my_warehouse
+    database: my_database
+    schema: my_schema      # Defaults to PUBLIC if not specified
+    role: my_role          # Optional
+```
+
+#### Connection Parameters
+
+- **account**: Your Snowflake account identifier (e.g., `myorg-account123`)
+- **user**: The username for authentication
+- **password**: The password for authentication
+- **warehouse**: The compute warehouse to use (required)
+- **database**: The database where state will be stored
+- **schema**: The schema where state tables will be created (defaults to PUBLIC)
+- **role**: Optional role to use for the connection
+
+#### Security Considerations
+
+When storing credentials:
+- Use environment variables for sensitive values in production
+- Consider using Snowflake key-pair authentication (future enhancement)
+- Ensure the user has CREATE TABLE, INSERT, UPDATE, DELETE, and SELECT privileges
+
+Example using environment variables:
+
+```bash
+export MELTANO_STATE_BACKEND_SNOWFLAKE_PASSWORD='my_secure_password'
+meltano config meltano set state_backend.uri 'snowflake://my_user@my_account/my_database'
+meltano config meltano set state_backend.snowflake.warehouse 'my_warehouse'
+```
+
+## Development
+
+### Setup
+
+```bash
+uv sync
+```
+
+### Run tests
+
+Run all tests, type checks, linting, and coverage:
+
+```bash
+uvx -with tox-uv tox run-parallel
+```
+
+### Bump the version
+
+```bash
+uv version --bump <type>
+```
+
+[meltano]: https://meltano.com
+[snowflake]: https://www.snowflake.com/
+[state-backend]: https://docs.meltano.com/concepts/state_backends
+[pipx]: https://github.com/pypa/pipx
+[uv]: https://docs.astral.sh/uv

meltano_state_backend_snowflake-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+meltano_state_backend_snowflake/__init__.py,sha256=VKrpkX_1iA77T_Akw03QYSwhEouiB6WCwk8ni2SAexM,43
+meltano_state_backend_snowflake/backend.py,sha256=f_dVCUG0YYy9z1Rl_oXEgDfBOgTfekTsRZsMmXddk9s,13639
+meltano_state_backend_snowflake/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+meltano_state_backend_snowflake-0.1.0.dist-info/METADATA,sha256=U5abP9PQjKPmB4b0Vo0wgJ6YLit_mE-u-jtDC2JOueY,3944
+meltano_state_backend_snowflake-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+meltano_state_backend_snowflake-0.1.0.dist-info/entry_points.txt,sha256=jZRNuruL8DV7LQshmT9rLCjFWFU1RFDiTKMjSdb1Gak,664
+meltano_state_backend_snowflake-0.1.0.dist-info/licenses/LICENSE,sha256=-5_wfLmGpH1fzTKoBkjWeEnK7cV60Cs0c6Ap3bE9n1U,1064
+meltano_state_backend_snowflake-0.1.0.dist-info/RECORD,,

meltano_state_backend_snowflake-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

meltano_state_backend_snowflake-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,11 @@
+[meltano.settings]
+snowflake_account = meltano_state_backend_snowflake.backend:SNOWFLAKE_ACCOUNT
+snowflake_database = meltano_state_backend_snowflake.backend:SNOWFLAKE_DATABASE
+snowflake_password = meltano_state_backend_snowflake.backend:SNOWFLAKE_PASSWORD
+snowflake_role = meltano_state_backend_snowflake.backend:SNOWFLAKE_ROLE
+snowflake_schema = meltano_state_backend_snowflake.backend:SNOWFLAKE_SCHEMA
+snowflake_user = meltano_state_backend_snowflake.backend:SNOWFLAKE_USER
+snowflake_warehouse = meltano_state_backend_snowflake.backend:SNOWFLAKE_WAREHOUSE
+
+[meltano.state_backends]
+snowflake = meltano_state_backend_snowflake.backend:SnowflakeStateStoreManager

meltano_state_backend_snowflake-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Meltano
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.