snowflake-cli-labs 2.7.0rc1__tar.gz → 2.7.0rc3__tar.gz

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snowflake-cli-labs
-Version: 2.7.0rc1
+Version: 2.7.0rc3
 Summary: Snowflake CLI
 Project-URL: Source code, https://github.com/snowflakedb/snowflake-cli
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowflake-cli/issues

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/RELEASE-NOTES.md

@@ -48,6 +48,7 @@ but should be replaced with `snow init`
 * Updated post_deploy SQL script default database to be the application database
 * Handle `NULL` md5 values correctly when returned by stage storage backends
 * Regionless host URLs are now supported when generating Snowsight URLs
+* `snow app run` and `snow app deploy` now correctly determine modified status for large files uploaded to AWS S3
 
 # v2.6.1
 ## Backward incompatibility

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/__about__.py

@@ -14,4 +14,4 @@
 
 from __future__ import annotations
 
-VERSION = "2.7.0rc1"
+VERSION = "2.7.0rc3"

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/api/config.py

@@ -78,6 +78,7 @@ class ConnectionConfig:
     role: Optional[str] = None
     authenticator: Optional[str] = None
     private_key_path: Optional[str] = None
+    token_file_path: Optional[str] = None
 
     _other_settings: dict = field(default_factory=lambda: {})
 

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/api/feature_flags.py

@@ -48,4 +48,9 @@ class FeatureFlag(FeatureFlagMixin):
     ENABLE_STREAMLIT_EMBEDDED_STAGE = BooleanFlag(
         "ENABLE_STREAMLIT_EMBEDDED_STAGE", False
     )
-    ENABLE_PROJECT_DEFINITION_V2 = BooleanFlag("ENABLE_PROJECT_DEFINITION_V2", False)
+    ENABLE_STREAMLIT_NO_CHECKOUTS = BooleanFlag("ENABLE_STREAMLIT_NO_CHECKOUTS", False)
+    ENABLE_STREAMLIT_VERSIONED_STAGE = BooleanFlag(
+        "ENABLE_STREAMLIT_VERSIONED_STAGE", False
+    )
+    # TODO: remove in 3.0
+    ENABLE_PROJECT_DEFINITION_V2 = BooleanFlag("ENABLE_PROJECT_DEFINITION_V2", True)

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/plugins/connection/commands.py

@@ -219,6 +219,14 @@ def add(
         prompt="Path to private key file",
         help="Path to file containing private key",
     ),
+    token_file_path: str = typer.Option(
+        EmptyInput(),
+        "--token-file-path",
+        "-t",
+        click_type=OptionalPrompt(),
+        prompt="Path to token file",
+        help="Path to file with an OAuth token that should be used when connecting to Snowflake",
+    ),
     set_as_default: bool = typer.Option(
         False,
         "--default",
@@ -246,6 +254,7 @@ def add(
             role=role,
             authenticator=authenticator,
             private_key_path=private_key_path,
+            token_file_path=token_file_path,
         ),
     )
     if set_as_default:

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/plugins/nativeapp/commands.py

@@ -63,6 +63,11 @@ from snowflake.cli.plugins.nativeapp.v2_conversions.v2_to_v1_decorator import (
     nativeapp_definition_v2_to_v1,
 )
 from snowflake.cli.plugins.nativeapp.version.commands import app as versions_app
+from snowflake.cli.plugins.stage.diff import (
+    DiffResult,
+    compute_stage_diff,
+    print_diff_to_console,
+)
 
 app = SnowTyperFactory(
     name="app",
@@ -169,6 +174,32 @@ def app_bundle(
     return MessageResult(f"Bundle generated at {manager.deploy_root}")
 
 
+@app.command("diff", requires_connection=True, hidden=True)
+@with_project_definition()
+@nativeapp_definition_v2_to_v1
+def app_diff(
+    **options,
+) -> CommandResult:
+    """
+    Performs a diff between the app's source stage and the local deploy root.
+    """
+    assert_project_type("native_app")
+
+    manager = NativeAppManager(
+        project_definition=cli_context.project_definition.native_app,
+        project_root=cli_context.project_root,
+    )
+    bundle_map = manager.build_bundle()
+    diff: DiffResult = compute_stage_diff(
+        local_root=Path(manager.deploy_root), stage_fqn=manager.stage_fqn
+    )
+    if cli_context.output_format == OutputFormat.JSON:
+        return ObjectResult(diff.to_dict())
+    else:
+        print_diff_to_console(diff, bundle_map)
+        return None  # don't print any output
+
+
 @app.command("run", requires_connection=True)
 @with_project_definition()
 @nativeapp_definition_v2_to_v1

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/plugins/stage/diff.py

@@ -14,9 +14,7 @@
 
 from __future__ import annotations
 
-import hashlib
 import logging
-import re
 from dataclasses import dataclass, field
 from pathlib import Path, PurePosixPath
 from typing import Collection, Dict, List, Optional, Tuple
@@ -25,14 +23,11 @@ from snowflake.cli.api.console import cli_console as cc
 from snowflake.cli.api.exceptions import (
     SnowflakeSQLExecutionError,
 )
-from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
 from snowflake.cli.plugins.nativeapp.artifacts import BundleMap
 from snowflake.connector.cursor import DictCursor
 
 from .manager import StageManager
-
-MD5SUM_REGEX = r"^[A-Fa-f0-9]{32}$"
-CHUNK_SIZE_BYTES = 8192
+from .md5 import UnknownMD5FormatError, file_matches_md5sum
 
 log = logging.getLogger(__name__)
 
@@ -72,45 +67,6 @@ class DiffResult:
         }
 
 
-def is_valid_md5sum(checksum: Optional[str]) -> bool:
-    """
-    Could the provided hexadecimal checksum represent a valid md5sum?
-    """
-    if checksum is None:
-        return False
-    return re.match(MD5SUM_REGEX, checksum) is not None
-
-
-def compute_md5sum(file: Path) -> str:
-    """
-    Returns a hexadecimal checksum for the file located at the given path.
-    """
-    if not file.is_file():
-        raise ValueError(
-            "The provided file does not exist or not a (symlink to a) regular file"
-        )
-
-    # FIXME: there are two cases in which this will fail to provide a matching
-    # md5sum, even when the underlying file is the same:
-    #  1. when the stage uses SNOWFLAKE_FULL encryption
-    #  2. when the file was uploaded in multiple parts
-
-    # We can re-create the second if we know what chunk size was used by the
-    # upload process to the backing object store (e.g. S3, azure blob, etc.)
-    # but we cannot re-create the first as the encrpytion key is hidden.
-
-    # We are assuming that we will not get accidental collisions here due to the
-    # large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance)
-    # combined with the fact that the file name + path must also match elsewhere.
-
-    with SecurePath(file).open("rb", read_file_limit_mb=UNLIMITED) as f:
-        file_hash = hashlib.md5()
-        while chunk := f.read(CHUNK_SIZE_BYTES):
-            file_hash.update(chunk)
-
-    return file_hash.hexdigest()
-
-
 def enumerate_files(path: Path) -> List[Path]:
     """
     Get a list of all files in a directory (recursively).
@@ -175,30 +131,35 @@ def compute_stage_diff(
 
     for local_file in local_files:
         relpath = local_file.relative_to(local_root)
-        stage_filename = to_stage_path(relpath)
-        if stage_filename not in remote_md5:
+        stage_path = to_stage_path(relpath)
+        if stage_path not in remote_md5:
             # doesn't exist on the stage
-            result.only_local.append(stage_filename)
+            result.only_local.append(stage_path)
         else:
-            # N.B. we could compare local size vs remote size to skip the relatively-
-            # expensive md5sum operation, but after seeing a comment that says the value
-            # may not always be correctly populated, we'll ignore that column.
-            stage_md5sum = remote_md5[stage_filename]
-            if is_valid_md5sum(stage_md5sum) and stage_md5sum == compute_md5sum(
-                local_file
-            ):
-                # the file definitely hasn't changed
-                result.identical.append(stage_filename)
-            else:
-                # either the file has changed, or we can't tell if it has
-                result.different.append(stage_filename)
+            # N.B. file size on stage is not always accurate, so cannot fail fast
+            try:
+                if file_matches_md5sum(local_file, remote_md5[stage_path]):
+                    # We are assuming that we will not get accidental collisions here due to the
+                    # large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance)
+                    # combined with the fact that the file name + path must also match elsewhere.
+                    result.identical.append(stage_path)
+                else:
+                    # either the file has changed, or we can't tell if it has
+                    result.different.append(stage_path)
+            except UnknownMD5FormatError:
+                log.warning(
+                    "Could not compare md5 for %s, assuming file has changed",
+                    local_file,
+                    exc_info=True,
+                )
+                result.different.append(stage_path)
 
             # mark this file as seen
-            del remote_md5[stage_filename]
+            del remote_md5[stage_path]
 
     # every entry here is a file we never saw locally
-    for stage_filename in remote_md5.keys():
-        result.only_on_stage.append(stage_filename)
+    for stage_path in remote_md5.keys():
+        result.only_on_stage.append(stage_path)
 
     return result
 
@@ -232,8 +193,8 @@ def delete_only_on_stage_files(
     """
     Deletes all files from a Snowflake stage according to the input list of filenames, using a custom role.
     """
-    for _stage_filename in only_on_stage:
-        stage_manager.remove(stage_name=stage_fqn, path=str(_stage_filename), role=role)
+    for _stage_path in only_on_stage:
+        stage_manager.remove(stage_name=stage_fqn, path=str(_stage_path), role=role)
 
 
 def put_files_on_stage(

snowflake_cli_labs-2.7.0rc3/src/snowflake/cli/plugins/stage/md5.py

@@ -0,0 +1,160 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+import os.path
+import re
+from pathlib import Path
+from typing import List, Tuple
+
+from click.exceptions import ClickException
+from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
+from snowflake.connector.constants import S3_CHUNK_SIZE, S3_MAX_PARTS, S3_MIN_PART_SIZE
+
+ONE_MEGABYTE = 1024**2
+READ_BUFFER_BYTES = 64 * 1024
+MD5SUM_REGEX = r"^[A-Fa-f0-9]{32}$"
+MULTIPART_MD5SUM_REGEX = r"^([A-Fa-f0-9]{32})-(\d+)$"
+
+log = logging.getLogger(__name__)
+
+
+class UnknownMD5FormatError(ClickException):
+    def __init__(self, md5: str):
+        super().__init__(f"Unknown md5 format: {md5}")
+
+
+def is_md5sum(checksum: str) -> bool:
+    """
+    Could the provided hexadecimal checksum represent a valid md5sum?
+    """
+    return re.match(MD5SUM_REGEX, checksum) is not None
+
+
+def parse_multipart_md5sum(checksum: str) -> Tuple[str, int] | None:
+    """
+    Does this represent a multi-part md5sum (i.e. "<md5>-<n>")?
+    If so, returns the tuple (md5, n), otherwise None.
+    """
+    multipart_md5 = re.match(MULTIPART_MD5SUM_REGEX, checksum)
+    if multipart_md5:
+        return (multipart_md5.group(1), int(multipart_md5.group(2)))
+    return None
+
+
+def compute_md5sum(file: Path, chunk_size: int | None = None) -> str:
+    """
+    Returns a hexadecimal checksum for the file located at the given path.
+    If chunk_size is given, computes a multi-part md5sum.
+    """
+    if not file.is_file():
+        raise ValueError(
+            "The provided file does not exist or not a (symlink to a) regular file"
+        )
+
+    # If the stage uses SNOWFLAKE_FULL encryption, this will fail to provide
+    # a matching md5sum, even when the underlying file is the same, as we do
+    # not have access to the encrypted file under checksum.
+
+    file_size = os.path.getsize(file)
+    if file_size == 0:
+        # simple md5 with no content
+        return hashlib.md5().hexdigest()
+
+    with SecurePath(file).open("rb", read_file_limit_mb=UNLIMITED) as f:
+        md5s: List[hashlib._Hash] = []  # noqa: SLF001
+        hasher = hashlib.md5()
+
+        remains = file_size
+        remains_in_chunk: int = min(chunk_size, remains) if chunk_size else remains
+        while remains > 0:
+            sz = min(READ_BUFFER_BYTES, remains_in_chunk)
+            buf = f.read(sz)
+            hasher.update(buf)
+            remains_in_chunk -= sz
+            remains -= sz
+            if remains_in_chunk == 0:
+                if not chunk_size:
+                    # simple md5; only one chunk processed
+                    return hasher.hexdigest()
+                else:
+                    # push the hash of this chunk + reset
+                    md5s.append(hasher)
+                    hasher = hashlib.md5()
+                    remains_in_chunk = min(chunk_size, remains)
+
+        # multi-part hash (e.g. aws)
+        digests = b"".join(m.digest() for m in md5s)
+        digests_md5 = hashlib.md5(digests)
+        return f"{digests_md5.hexdigest()}-{len(md5s)}"
+
+
+def file_matches_md5sum(local_file: Path, remote_md5: str | None) -> bool:
+    """
+    Try a few different md5sums to determine if a local file is identical
+    to a file that has a given remote md5sum.
+
+    Handles the multi-part md5sums generated by e.g. AWS S3, using values
+    from the python connector to make educated guesses on chunk size.
+
+    Assumes that upload time would dominate local hashing time.
+    """
+    if not remote_md5:
+        # no hash available
+        return False
+
+    if is_md5sum(remote_md5):
+        # regular hash
+        return compute_md5sum(local_file) == remote_md5
+
+    if md5_and_chunks := parse_multipart_md5sum(remote_md5):
+        # multi-part hash (e.g. aws)
+        (_, num_chunks) = md5_and_chunks
+        file_size = os.path.getsize(local_file)
+
+        # If this file uses the maximum number of parts supported by the cloud backend,
+        # the chunk size is likely not a clean multiple of a megabyte. Try reverse engineering
+        # from the file size first, then fall back to the usual detection method.
+        # At time of writing this logic would trigger for files >= 80GiB (python connector)
+        if num_chunks == S3_MAX_PARTS:
+            chunk_size = max(math.ceil(file_size / S3_MAX_PARTS), S3_MIN_PART_SIZE)
+            if compute_md5sum(local_file, chunk_size) == remote_md5:
+                return True
+
+        # Estimates the chunk size the multi-part file must have been uploaded with
+        # by trying chunk sizes that give the most evenly-sized chunks.
+        #
+        # First we'll try the chunk size that's a multiple of S3_CHUNK_SIZE (8mb) from
+        # the python connector that results in num_chunks, then we'll do the same with
+        # a smaller granularity (1mb) that is used by default in some AWS multi-part
+        # upload implementations.
+        #
+        # We're working backwards from num_chunks here because it's the only value we know.
+        for chunk_size_alignment in [S3_CHUNK_SIZE, ONE_MEGABYTE]:
+            # +1 because we need at least one chunk when file_size < num_chunks * chunk_size_alignment
+            # -1 because we don't want to add an extra chunk when file_size is an exact multiple of num_chunks * chunk_size_alignment
+            multiplier = 1 + ((file_size - 1) // (num_chunks * chunk_size_alignment))
+            chunk_size = multiplier * chunk_size_alignment
+            if compute_md5sum(local_file, chunk_size) == remote_md5:
+                return True
+
+        # we were unable to figure out the chunk size, or the files are different
+        log.debug("multi-part md5: %s != %s", remote_md5, local_file)
+        return False
+
+    raise UnknownMD5FormatError(remote_md5)

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/src/snowflake/cli/plugins/streamlit/manager.py

@@ -121,10 +121,11 @@ class StreamlitManager(SqlExecutionMixin):
         # for backwards compatibility - quoted stage path might be case-sensitive
         # https://docs.snowflake.com/en/sql-reference/identifiers-syntax#double-quoted-identifiers
         streamlit_name_for_root_location = streamlit_id.name
-
+        use_versioned_stage = FeatureFlag.ENABLE_STREAMLIT_VERSIONED_STAGE.is_enabled()
         if (
             experimental_behaviour_enabled()
             or FeatureFlag.ENABLE_STREAMLIT_EMBEDDED_STAGE.is_enabled()
+            or use_versioned_stage
         ):
             """
             1. Create streamlit object
@@ -141,19 +142,30 @@ class StreamlitManager(SqlExecutionMixin):
                 title=title,
             )
             try:
-                self._execute_query(
-                    f"ALTER streamlit {streamlit_id.identifier} CHECKOUT"
-                )
+                if use_versioned_stage:
+                    self._execute_query(
+                        f"ALTER STREAMLIT {streamlit_id.identifier} ADD LIVE VERSION FROM LAST"
+                    )
+                elif not FeatureFlag.ENABLE_STREAMLIT_NO_CHECKOUTS.is_enabled():
+                    self._execute_query(
+                        f"ALTER streamlit {streamlit_id.identifier} CHECKOUT"
+                    )
             except ProgrammingError as e:
-                # If an error is raised because a CHECKOUT has already occurred,
-                # simply skip it and continue
-                if "Checkout already exists" in str(e):
+                # If an error is raised because a CHECKOUT has already occurred or a LIVE VERSION already exists, simply skip it and continue
+                if "Checkout already exists" in str(
+                    e
+                ) or "There is already a live version" in str(e):
                     log.info("Checkout already exists, continuing")
                 else:
                     raise
+
             stage_path = streamlit_id.identifier
             embedded_stage_name = f"snow://streamlit/{stage_path}"
-            root_location = f"{embedded_stage_name}/default_checkout"
+            if use_versioned_stage:
+                # "LIVE" is the only supported version for now, but this may change later.
+                root_location = f"{embedded_stage_name}/versions/live"
+            else:
+                root_location = f"{embedded_stage_name}/default_checkout"
 
             self._put_streamlit_files(
                 root_location,

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/tests/__snapshots__/test_help_messages.ambr

@@ -171,6 +171,78 @@
   +------------------------------------------------------------------------------+
   
   
+  '''
+# ---
+# name: test_help_messages[app.diff]
+  '''
+                                                                                  
+   Usage: default app diff [OPTIONS]                                              
+                                                                                  
+   Performs a diff between the app's source stage and the local deploy root.      
+                                                                                  
+  +- Options --------------------------------------------------------------------+
+  | --project  -p      TEXT  Path where Snowflake project resides. Defaults to   |
+  |                          current working directory.                          |
+  | --env              TEXT  String in format of key=value. Overrides variables  |
+  |                          from env section used for templating.               |
+  | --help     -h            Show this message and exit.                         |
+  +------------------------------------------------------------------------------+
+  +- Connection configuration ---------------------------------------------------+
+  | --connection,--environment  -c      TEXT  Name of the connection, as defined |
+  |                                           in your `config.toml`. Default:    |
+  |                                           `default`.                         |
+  | --account,--accountname             TEXT  Name assigned to your Snowflake    |
+  |                                           account. Overrides the value       |
+  |                                           specified for the connection.      |
+  | --user,--username                   TEXT  Username to connect to Snowflake.  |
+  |                                           Overrides the value specified for  |
+  |                                           the connection.                    |
+  | --password                          TEXT  Snowflake password. Overrides the  |
+  |                                           value specified for the            |
+  |                                           connection.                        |
+  | --authenticator                     TEXT  Snowflake authenticator. Overrides |
+  |                                           the value specified for the        |
+  |                                           connection.                        |
+  | --private-key-path                  TEXT  Snowflake private key path.        |
+  |                                           Overrides the value specified for  |
+  |                                           the connection.                    |
+  | --token-file-path                   TEXT  Path to file with an OAuth token   |
+  |                                           that should be used when           |
+  |                                           connecting to Snowflake            |
+  | --database,--dbname                 TEXT  Database to use. Overrides the     |
+  |                                           value specified for the            |
+  |                                           connection.                        |
+  | --schema,--schemaname               TEXT  Database schema to use. Overrides  |
+  |                                           the value specified for the        |
+  |                                           connection.                        |
+  | --role,--rolename                   TEXT  Role to use. Overrides the value   |
+  |                                           specified for the connection.      |
+  | --warehouse                         TEXT  Warehouse to use. Overrides the    |
+  |                                           value specified for the            |
+  |                                           connection.                        |
+  | --temporary-connection      -x            Uses connection defined with       |
+  |                                           command line parameters, instead   |
+  |                                           of one defined in config           |
+  | --mfa-passcode                      TEXT  Token to use for multi-factor      |
+  |                                           authentication (MFA)               |
+  | --enable-diag                             Run python connector diagnostic    |
+  |                                           test                               |
+  | --diag-log-path                     TEXT  Diagnostic report path             |
+  | --diag-allowlist-path               TEXT  Diagnostic report path to optional |
+  |                                           allowlist                          |
+  +------------------------------------------------------------------------------+
+  +- Global configuration -------------------------------------------------------+
+  | --format           [TABLE|JSON]  Specifies the output format.                |
+  |                                  [default: TABLE]                            |
+  | --verbose  -v                    Displays log entries for log levels `info`  |
+  |                                  and higher.                                 |
+  | --debug                          Displays log entries for log levels `debug` |
+  |                                  and higher; debug logs contains additional  |
+  |                                  information.                                |
+  | --silent                         Turns off intermediate output to console.   |
+  +------------------------------------------------------------------------------+
+  
+  
   '''
 # ---
 # name: test_help_messages[app.events]
@@ -1106,6 +1178,10 @@
   |                                        [default: optional]                   |
   | --private-key            -k      TEXT  Path to file containing private key   |
   |                                        [default: optional]                   |
+  | --token-file-path        -t      TEXT  Path to file with an OAuth token that |
+  |                                        should be used when connecting to     |
+  |                                        Snowflake                             |
+  |                                        [default: optional]                   |
   | --default                              If provided the connection will be    |
   |                                        configured as default connection.     |
   | --help                                 Show this message and exit.           |

{snowflake_cli_labs-2.7.0rc1 → snowflake_cli_labs-2.7.0rc3}/tests/project/test_config.py

@@ -167,5 +167,7 @@ def test_project_definition_v2_is_disabled():
 def test_project_definition_v2_is_enabled_with_feature_flag():
     with mock_config_key("enable_project_definition_v2", True):
         assert FeatureFlag.ENABLE_STREAMLIT_EMBEDDED_STAGE.is_enabled() == False
+        assert FeatureFlag.ENABLE_STREAMLIT_NO_CHECKOUTS.is_enabled() == False
+        assert FeatureFlag.ENABLE_STREAMLIT_VERSIONED_STAGE.is_enabled() == False
         assert FeatureFlag.ENABLE_PROJECT_DEFINITION_V2.is_enabled() == True
         build_project_definition(**{"definition_version": "2", "entities": {}})