snowflake-cli-labs 2.7.0rc2__py3-none-any.whl → 2.7.0rc3__py3-none-any.whl

snowflake/cli/__about__.py

@@ -14,4 +14,4 @@
 
 from __future__ import annotations
 
-VERSION = "2.7.0rc2"
+VERSION = "2.7.0rc3"

snowflake/cli/plugins/nativeapp/commands.py

@@ -63,6 +63,11 @@ from snowflake.cli.plugins.nativeapp.v2_conversions.v2_to_v1_decorator import (
     nativeapp_definition_v2_to_v1,
 )
 from snowflake.cli.plugins.nativeapp.version.commands import app as versions_app
+from snowflake.cli.plugins.stage.diff import (
+    DiffResult,
+    compute_stage_diff,
+    print_diff_to_console,
+)
 
 app = SnowTyperFactory(
     name="app",
@@ -169,6 +174,32 @@ def app_bundle(
     return MessageResult(f"Bundle generated at {manager.deploy_root}")
 
 
+@app.command("diff", requires_connection=True, hidden=True)
+@with_project_definition()
+@nativeapp_definition_v2_to_v1
+def app_diff(
+    **options,
+) -> CommandResult:
+    """
+    Performs a diff between the app's source stage and the local deploy root.
+    """
+    assert_project_type("native_app")
+
+    manager = NativeAppManager(
+        project_definition=cli_context.project_definition.native_app,
+        project_root=cli_context.project_root,
+    )
+    bundle_map = manager.build_bundle()
+    diff: DiffResult = compute_stage_diff(
+        local_root=Path(manager.deploy_root), stage_fqn=manager.stage_fqn
+    )
+    if cli_context.output_format == OutputFormat.JSON:
+        return ObjectResult(diff.to_dict())
+    else:
+        print_diff_to_console(diff, bundle_map)
+        return None  # don't print any output
+
+
 @app.command("run", requires_connection=True)
 @with_project_definition()
 @nativeapp_definition_v2_to_v1

snowflake/cli/plugins/stage/diff.py

@@ -14,9 +14,7 @@
 
 from __future__ import annotations
 
-import hashlib
 import logging
-import re
 from dataclasses import dataclass, field
 from pathlib import Path, PurePosixPath
 from typing import Collection, Dict, List, Optional, Tuple
@@ -25,14 +23,11 @@ from snowflake.cli.api.console import cli_console as cc
 from snowflake.cli.api.exceptions import (
     SnowflakeSQLExecutionError,
 )
-from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
 from snowflake.cli.plugins.nativeapp.artifacts import BundleMap
 from snowflake.connector.cursor import DictCursor
 
 from .manager import StageManager
-
-MD5SUM_REGEX = r"^[A-Fa-f0-9]{32}$"
-CHUNK_SIZE_BYTES = 8192
+from .md5 import UnknownMD5FormatError, file_matches_md5sum
 
 log = logging.getLogger(__name__)
 
@@ -72,45 +67,6 @@ class DiffResult:
         }
 
 
-def is_valid_md5sum(checksum: Optional[str]) -> bool:
-    """
-    Could the provided hexadecimal checksum represent a valid md5sum?
-    """
-    if checksum is None:
-        return False
-    return re.match(MD5SUM_REGEX, checksum) is not None
-
-
-def compute_md5sum(file: Path) -> str:
-    """
-    Returns a hexadecimal checksum for the file located at the given path.
-    """
-    if not file.is_file():
-        raise ValueError(
-            "The provided file does not exist or not a (symlink to a) regular file"
-        )
-
-    # FIXME: there are two cases in which this will fail to provide a matching
-    # md5sum, even when the underlying file is the same:
-    #  1. when the stage uses SNOWFLAKE_FULL encryption
-    #  2. when the file was uploaded in multiple parts
-
-    # We can re-create the second if we know what chunk size was used by the
-    # upload process to the backing object store (e.g. S3, azure blob, etc.)
-    # but we cannot re-create the first as the encrpytion key is hidden.
-
-    # We are assuming that we will not get accidental collisions here due to the
-    # large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance)
-    # combined with the fact that the file name + path must also match elsewhere.
-
-    with SecurePath(file).open("rb", read_file_limit_mb=UNLIMITED) as f:
-        file_hash = hashlib.md5()
-        while chunk := f.read(CHUNK_SIZE_BYTES):
-            file_hash.update(chunk)
-
-    return file_hash.hexdigest()
-
-
 def enumerate_files(path: Path) -> List[Path]:
     """
     Get a list of all files in a directory (recursively).
@@ -175,30 +131,35 @@ def compute_stage_diff(
 
     for local_file in local_files:
         relpath = local_file.relative_to(local_root)
-        stage_filename = to_stage_path(relpath)
-        if stage_filename not in remote_md5:
+        stage_path = to_stage_path(relpath)
+        if stage_path not in remote_md5:
             # doesn't exist on the stage
-            result.only_local.append(stage_filename)
+            result.only_local.append(stage_path)
         else:
-            # N.B. we could compare local size vs remote size to skip the relatively-
-            # expensive md5sum operation, but after seeing a comment that says the value
-            # may not always be correctly populated, we'll ignore that column.
-            stage_md5sum = remote_md5[stage_filename]
-            if is_valid_md5sum(stage_md5sum) and stage_md5sum == compute_md5sum(
-                local_file
-            ):
-                # the file definitely hasn't changed
-                result.identical.append(stage_filename)
-            else:
-                # either the file has changed, or we can't tell if it has
-                result.different.append(stage_filename)
+            # N.B. file size on stage is not always accurate, so cannot fail fast
+            try:
+                if file_matches_md5sum(local_file, remote_md5[stage_path]):
+                    # We are assuming that we will not get accidental collisions here due to the
+                    # large space of the md5sum (32 * 4 = 128 bits means 1-in-9-trillion chance)
+                    # combined with the fact that the file name + path must also match elsewhere.
+                    result.identical.append(stage_path)
+                else:
+                    # either the file has changed, or we can't tell if it has
+                    result.different.append(stage_path)
+            except UnknownMD5FormatError:
+                log.warning(
+                    "Could not compare md5 for %s, assuming file has changed",
+                    local_file,
+                    exc_info=True,
+                )
+                result.different.append(stage_path)
 
             # mark this file as seen
-            del remote_md5[stage_filename]
+            del remote_md5[stage_path]
 
     # every entry here is a file we never saw locally
-    for stage_filename in remote_md5.keys():
-        result.only_on_stage.append(stage_filename)
+    for stage_path in remote_md5.keys():
+        result.only_on_stage.append(stage_path)
 
     return result
 
@@ -232,8 +193,8 @@ def delete_only_on_stage_files(
     """
     Deletes all files from a Snowflake stage according to the input list of filenames, using a custom role.
     """
-    for _stage_filename in only_on_stage:
-        stage_manager.remove(stage_name=stage_fqn, path=str(_stage_filename), role=role)
+    for _stage_path in only_on_stage:
+        stage_manager.remove(stage_name=stage_fqn, path=str(_stage_path), role=role)
 
 
 def put_files_on_stage(

snowflake/cli/plugins/stage/md5.py

@@ -0,0 +1,160 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import math
+import os.path
+import re
+from pathlib import Path
+from typing import List, Tuple
+
+from click.exceptions import ClickException
+from snowflake.cli.api.secure_path import UNLIMITED, SecurePath
+from snowflake.connector.constants import S3_CHUNK_SIZE, S3_MAX_PARTS, S3_MIN_PART_SIZE
+
+ONE_MEGABYTE = 1024**2
+READ_BUFFER_BYTES = 64 * 1024
+MD5SUM_REGEX = r"^[A-Fa-f0-9]{32}$"
+MULTIPART_MD5SUM_REGEX = r"^([A-Fa-f0-9]{32})-(\d+)$"
+
+log = logging.getLogger(__name__)
+
+
+class UnknownMD5FormatError(ClickException):
+    def __init__(self, md5: str):
+        super().__init__(f"Unknown md5 format: {md5}")
+
+
+def is_md5sum(checksum: str) -> bool:
+    """
+    Could the provided hexadecimal checksum represent a valid md5sum?
+    """
+    return re.match(MD5SUM_REGEX, checksum) is not None
+
+
+def parse_multipart_md5sum(checksum: str) -> Tuple[str, int] | None:
+    """
+    Does this represent a multi-part md5sum (i.e. "<md5>-<n>")?
+    If so, returns the tuple (md5, n), otherwise None.
+    """
+    multipart_md5 = re.match(MULTIPART_MD5SUM_REGEX, checksum)
+    if multipart_md5:
+        return (multipart_md5.group(1), int(multipart_md5.group(2)))
+    return None
+
+
+def compute_md5sum(file: Path, chunk_size: int | None = None) -> str:
+    """
+    Returns a hexadecimal checksum for the file located at the given path.
+    If chunk_size is given, computes a multi-part md5sum.
+    """
+    if not file.is_file():
+        raise ValueError(
+            "The provided file does not exist or not a (symlink to a) regular file"
+        )
+
+    # If the stage uses SNOWFLAKE_FULL encryption, this will fail to provide
+    # a matching md5sum, even when the underlying file is the same, as we do
+    # not have access to the encrypted file under checksum.
+
+    file_size = os.path.getsize(file)
+    if file_size == 0:
+        # simple md5 with no content
+        return hashlib.md5().hexdigest()
+
+    with SecurePath(file).open("rb", read_file_limit_mb=UNLIMITED) as f:
+        md5s: List[hashlib._Hash] = []  # noqa: SLF001
+        hasher = hashlib.md5()
+
+        remains = file_size
+        remains_in_chunk: int = min(chunk_size, remains) if chunk_size else remains
+        while remains > 0:
+            sz = min(READ_BUFFER_BYTES, remains_in_chunk)
+            buf = f.read(sz)
+            hasher.update(buf)
+            remains_in_chunk -= sz
+            remains -= sz
+            if remains_in_chunk == 0:
+                if not chunk_size:
+                    # simple md5; only one chunk processed
+                    return hasher.hexdigest()
+                else:
+                    # push the hash of this chunk + reset
+                    md5s.append(hasher)
+                    hasher = hashlib.md5()
+                    remains_in_chunk = min(chunk_size, remains)
+
+        # multi-part hash (e.g. aws)
+        digests = b"".join(m.digest() for m in md5s)
+        digests_md5 = hashlib.md5(digests)
+        return f"{digests_md5.hexdigest()}-{len(md5s)}"
+
+
+def file_matches_md5sum(local_file: Path, remote_md5: str | None) -> bool:
+    """
+    Try a few different md5sums to determine if a local file is identical
+    to a file that has a given remote md5sum.
+
+    Handles the multi-part md5sums generated by e.g. AWS S3, using values
+    from the python connector to make educated guesses on chunk size.
+
+    Assumes that upload time would dominate local hashing time.
+    """
+    if not remote_md5:
+        # no hash available
+        return False
+
+    if is_md5sum(remote_md5):
+        # regular hash
+        return compute_md5sum(local_file) == remote_md5
+
+    if md5_and_chunks := parse_multipart_md5sum(remote_md5):
+        # multi-part hash (e.g. aws)
+        (_, num_chunks) = md5_and_chunks
+        file_size = os.path.getsize(local_file)
+
+        # If this file uses the maximum number of parts supported by the cloud backend,
+        # the chunk size is likely not a clean multiple of a megabyte. Try reverse engineering
+        # from the file size first, then fall back to the usual detection method.
+        # At time of writing this logic would trigger for files >= 80GiB (python connector)
+        if num_chunks == S3_MAX_PARTS:
+            chunk_size = max(math.ceil(file_size / S3_MAX_PARTS), S3_MIN_PART_SIZE)
+            if compute_md5sum(local_file, chunk_size) == remote_md5:
+                return True
+
+        # Estimates the chunk size the multi-part file must have been uploaded with
+        # by trying chunk sizes that give the most evenly-sized chunks.
+        #
+        # First we'll try the chunk size that's a multiple of S3_CHUNK_SIZE (8mb) from
+        # the python connector that results in num_chunks, then we'll do the same with
+        # a smaller granularity (1mb) that is used by default in some AWS multi-part
+        # upload implementations.
+        #
+        # We're working backwards from num_chunks here because it's the only value we know.
+        for chunk_size_alignment in [S3_CHUNK_SIZE, ONE_MEGABYTE]:
+            # +1 because we need at least one chunk when file_size < num_chunks * chunk_size_alignment
+            # -1 because we don't want to add an extra chunk when file_size is an exact multiple of num_chunks * chunk_size_alignment
+            multiplier = 1 + ((file_size - 1) // (num_chunks * chunk_size_alignment))
+            chunk_size = multiplier * chunk_size_alignment
+            if compute_md5sum(local_file, chunk_size) == remote_md5:
+                return True
+
+        # we were unable to figure out the chunk size, or the files are different
+        log.debug("multi-part md5: %s != %s", remote_md5, local_file)
+        return False
+
+    raise UnknownMD5FormatError(remote_md5)

{snowflake_cli_labs-2.7.0rc2.dist-info → snowflake_cli_labs-2.7.0rc3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snowflake-cli-labs
-Version: 2.7.0rc2
+Version: 2.7.0rc3
 Summary: Snowflake CLI
 Project-URL: Source code, https://github.com/snowflakedb/snowflake-cli
 Project-URL: Bug Tracker, https://github.com/snowflakedb/snowflake-cli/issues

{snowflake_cli_labs-2.7.0rc2.dist-info → snowflake_cli_labs-2.7.0rc3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-snowflake/cli/__about__.py,sha256=Uv-6PLfi16rouu4FmVaYcQxzWlnqxuynvd1zCdCUdao,636
+snowflake/cli/__about__.py,sha256=GeHBgomHUvUVoWB0_VtuL0utqdut5R7zsZY61FBjzJU,636
 snowflake/cli/__init__.py,sha256=uGA_QRGW3iGwaegpFsLgOhup0zBliBSXh9ou8J439uU,578
 snowflake/cli/api/__init__.py,sha256=kD6lYv5et7QJvW7vzvLN9p2ibfD7pjh9KRWsp2QoYqo,1330
 snowflake/cli/api/cli_global_context.py,sha256=gAs7snaqRi5ESaxU8HcC_QBR2q9y1PMdhi7kQCGkICs,11714
@@ -125,7 +125,7 @@ snowflake/cli/plugins/init/commands.py,sha256=wLIuMlHFMBmS3oJWSLxYlMO0Ac1b9dBcZY
 snowflake/cli/plugins/init/plugin_spec.py,sha256=uxglpV4GxY_hysq5fit_XR8JWLGkA8sClEN0bAU5OwU,993
 snowflake/cli/plugins/nativeapp/__init__.py,sha256=uGA_QRGW3iGwaegpFsLgOhup0zBliBSXh9ou8J439uU,578
 snowflake/cli/plugins/nativeapp/artifacts.py,sha256=otpxa6YFWJs1W1KDzo1agAyvq5YqAR3YoLjJJhm1CLs,30178
-snowflake/cli/plugins/nativeapp/commands.py,sha256=86vCJmiVXHN49--Bu83FHZJamNoPLH5Oz_4HRiPuctQ,14612
+snowflake/cli/plugins/nativeapp/commands.py,sha256=e0JIWSeCRD5H33UBydlmXmIxNHnA2UakxnSbwNRS9sc,15551
 snowflake/cli/plugins/nativeapp/common_flags.py,sha256=7OUXprC2n3B0a9gGTj4AezHaJyt8-C7vgRI5q9l-UXE,1610
 snowflake/cli/plugins/nativeapp/constants.py,sha256=j25fS9dS54GPPp41njUOZTDykhYq12PY67B084FLCZk,956
 snowflake/cli/plugins/nativeapp/exceptions.py,sha256=Wh-qJlAG9UMdWB0lqAVafbBdA_hj_m7UnI4efLjOgUA,4360
@@ -207,8 +207,9 @@ snowflake/cli/plugins/sql/plugin_spec.py,sha256=U6ex88D6rhVkKR1d_HmVXo8EK0k9oZB-
 snowflake/cli/plugins/sql/snowsql_templating.py,sha256=VTPvFzZihZnu6R5L1JMaVdzxfviSRoMLlNyxK87sE5E,881
 snowflake/cli/plugins/stage/__init__.py,sha256=uGA_QRGW3iGwaegpFsLgOhup0zBliBSXh9ou8J439uU,578
 snowflake/cli/plugins/stage/commands.py,sha256=iLtu5qIQ7e1Ld-PaQ8DpuS5SP7C3q337dmlJIvriYUw,8201
-snowflake/cli/plugins/stage/diff.py,sha256=Wja12l-ORhBYsoKfr64ToowJLpsH7zaBowR7QuZGF6Y,12071
+snowflake/cli/plugins/stage/diff.py,sha256=xmZoZL3ZQZS36M5H8FEb29kRNpcE6KM7tJVw8ZRcb_A,10756
 snowflake/cli/plugins/stage/manager.py,sha256=r93oM5g-2rpXM7F7qtW00KlTmwCxKi7374ERvDgVHz8,20412
+snowflake/cli/plugins/stage/md5.py,sha256=hX_Ao7ys7hUPNuWLm5KEmIpcIULEIJ1UMgc4qUyvoLE,6303
 snowflake/cli/plugins/stage/plugin_spec.py,sha256=r8fvJxonf1TYBUm2361ka9fpsnA35NQRbGYqhKVfaC4,993
 snowflake/cli/plugins/streamlit/__init__.py,sha256=uGA_QRGW3iGwaegpFsLgOhup0zBliBSXh9ou8J439uU,578
 snowflake/cli/plugins/streamlit/commands.py,sha256=e6wkD2HyWH39S05q6YlHzvzLAhyph6gmltHxuWOc8ao,5975
@@ -230,8 +231,8 @@ snowflake/cli/templates/default_streamlit/snowflake.yml,sha256=yWFU-vqJ7Z17K3loU
 snowflake/cli/templates/default_streamlit/streamlit_app.py,sha256=hfYtJl4Rtm0n3J2gNeDwMT-leeGL5R-qJKwyJ0kUxAI,109
 snowflake/cli/templates/default_streamlit/common/hello.py,sha256=3Zt2LthAYDs6UGqOvRNCzYH-HISLHxdx_uAVhcCOtJM,37
 snowflake/cli/templates/default_streamlit/pages/my_page.py,sha256=f__P9j5XCo8J1c6du25wSvknIKvhTFWrXF_298YiQbw,49
-snowflake_cli_labs-2.7.0rc2.dist-info/METADATA,sha256=098gUXYelbDm6UjtsbMOhcNOyl5YdPSfL1r2dO7f9Pg,17804
-snowflake_cli_labs-2.7.0rc2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-snowflake_cli_labs-2.7.0rc2.dist-info/entry_points.txt,sha256=_qdnT44fYFbH78kb6Em5jr2_26amIg3UIAvSdmqT6TY,57
-snowflake_cli_labs-2.7.0rc2.dist-info/licenses/LICENSE,sha256=mJMA3Uz2AbjU_kVggo1CAx01XhBsI7BSi2H7ggUg_-c,11344
-snowflake_cli_labs-2.7.0rc2.dist-info/RECORD,,
+snowflake_cli_labs-2.7.0rc3.dist-info/METADATA,sha256=ClpceWcdart1yjS0ouF9lhDW5Q3XBepAXKMRMgcmQw8,17804
+snowflake_cli_labs-2.7.0rc3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+snowflake_cli_labs-2.7.0rc3.dist-info/entry_points.txt,sha256=_qdnT44fYFbH78kb6Em5jr2_26amIg3UIAvSdmqT6TY,57
+snowflake_cli_labs-2.7.0rc3.dist-info/licenses/LICENSE,sha256=mJMA3Uz2AbjU_kVggo1CAx01XhBsI7BSi2H7ggUg_-c,11344
+snowflake_cli_labs-2.7.0rc3.dist-info/RECORD,,