snowflake-cli-labs 2.4.1__py3-none-any.whl → 2.5.0rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. snowflake/cli/__about__.py +15 -1
  2. snowflake/cli/__init__.py +13 -0
  3. snowflake/cli/api/__init__.py +14 -0
  4. snowflake/cli/api/cli_global_context.py +26 -0
  5. snowflake/cli/api/commands/__init__.py +13 -0
  6. snowflake/cli/api/commands/alias.py +14 -0
  7. snowflake/cli/api/commands/decorators.py +14 -0
  8. snowflake/cli/api/commands/experimental_behaviour.py +14 -0
  9. snowflake/cli/api/commands/flags.py +21 -1
  10. snowflake/cli/api/commands/project_initialisation.py +19 -2
  11. snowflake/cli/api/commands/snow_typer.py +90 -1
  12. snowflake/cli/api/config.py +14 -0
  13. snowflake/cli/api/console/__init__.py +14 -0
  14. snowflake/cli/api/console/abc.py +14 -0
  15. snowflake/cli/api/console/console.py +14 -0
  16. snowflake/cli/api/console/enum.py +14 -0
  17. snowflake/cli/api/constants.py +14 -0
  18. snowflake/cli/api/exceptions.py +22 -0
  19. snowflake/cli/api/feature_flags.py +14 -1
  20. snowflake/cli/api/identifiers.py +16 -2
  21. snowflake/cli/api/output/__init__.py +13 -0
  22. snowflake/cli/api/output/formats.py +14 -0
  23. snowflake/cli/api/output/types.py +14 -0
  24. snowflake/cli/api/plugins/__init__.py +13 -0
  25. snowflake/cli/api/plugins/command/__init__.py +14 -0
  26. snowflake/cli/api/plugins/command/plugin_hook_specs.py +14 -0
  27. snowflake/cli/api/plugins/plugin_config.py +14 -0
  28. snowflake/cli/api/project/__init__.py +13 -0
  29. snowflake/cli/api/project/definition.py +54 -8
  30. snowflake/cli/api/project/definition_manager.py +28 -2
  31. snowflake/cli/api/project/errors.py +14 -0
  32. snowflake/cli/api/project/schemas/__init__.py +13 -0
  33. snowflake/cli/api/project/schemas/identifier_model.py +14 -0
  34. snowflake/cli/api/project/schemas/native_app/__init__.py +13 -0
  35. snowflake/cli/api/project/schemas/native_app/application.py +14 -0
  36. snowflake/cli/api/project/schemas/native_app/native_app.py +35 -0
  37. snowflake/cli/api/project/schemas/native_app/package.py +14 -0
  38. snowflake/cli/api/project/schemas/native_app/path_mapping.py +15 -1
  39. snowflake/cli/api/project/schemas/project_definition.py +14 -0
  40. snowflake/cli/api/project/schemas/snowpark/__init__.py +13 -0
  41. snowflake/cli/api/project/schemas/snowpark/argument.py +14 -0
  42. snowflake/cli/api/project/schemas/snowpark/callable.py +14 -0
  43. snowflake/cli/api/project/schemas/snowpark/snowpark.py +14 -0
  44. snowflake/cli/api/project/schemas/streamlit/__init__.py +13 -0
  45. snowflake/cli/api/project/schemas/streamlit/streamlit.py +14 -0
  46. snowflake/cli/api/project/schemas/updatable_model.py +14 -0
  47. snowflake/cli/api/project/util.py +32 -3
  48. snowflake/cli/api/secure_path.py +59 -7
  49. snowflake/cli/api/secure_utils.py +14 -0
  50. snowflake/cli/api/sql_execution.py +14 -0
  51. snowflake/cli/api/utils/__init__.py +13 -0
  52. snowflake/cli/api/utils/cursor.py +14 -0
  53. snowflake/cli/api/utils/definition_rendering.py +268 -0
  54. snowflake/cli/api/utils/dict_utils.py +73 -0
  55. snowflake/cli/api/utils/error_handling.py +14 -0
  56. snowflake/cli/api/utils/graph.py +97 -0
  57. snowflake/cli/api/utils/models.py +14 -0
  58. snowflake/cli/api/utils/naming_utils.py +13 -0
  59. snowflake/cli/api/utils/path_utils.py +14 -0
  60. snowflake/cli/api/utils/rendering.py +21 -152
  61. snowflake/cli/api/utils/types.py +18 -1
  62. snowflake/cli/app/__init__.py +14 -0
  63. snowflake/cli/app/__main__.py +14 -0
  64. snowflake/cli/app/api_impl/__init__.py +13 -0
  65. snowflake/cli/app/api_impl/plugin/__init__.py +13 -0
  66. snowflake/cli/app/api_impl/plugin/plugin_config_provider_impl.py +14 -0
  67. snowflake/cli/app/cli_app.py +14 -0
  68. snowflake/cli/app/commands_registration/__init__.py +14 -0
  69. snowflake/cli/app/commands_registration/builtin_plugins.py +14 -0
  70. snowflake/cli/app/commands_registration/command_plugins_loader.py +14 -0
  71. snowflake/cli/app/commands_registration/commands_registration_with_callbacks.py +14 -0
  72. snowflake/cli/app/commands_registration/exception_logging.py +14 -0
  73. snowflake/cli/app/commands_registration/threadsafe.py +14 -0
  74. snowflake/cli/app/commands_registration/typer_registration.py +14 -0
  75. snowflake/cli/app/constants.py +14 -0
  76. snowflake/cli/app/dev/__init__.py +13 -0
  77. snowflake/cli/app/dev/commands_structure.py +14 -0
  78. snowflake/cli/app/dev/docs/__init__.py +13 -0
  79. snowflake/cli/app/dev/docs/generator.py +14 -0
  80. snowflake/cli/app/dev/pycharm_remote_debug.py +14 -0
  81. snowflake/cli/app/loggers.py +14 -0
  82. snowflake/cli/app/main_typer.py +14 -0
  83. snowflake/cli/app/printing.py +14 -0
  84. snowflake/cli/app/snow_connector.py +14 -0
  85. snowflake/cli/app/telemetry.py +14 -0
  86. snowflake/cli/plugins/__init__.py +13 -0
  87. snowflake/cli/plugins/connection/__init__.py +13 -0
  88. snowflake/cli/plugins/connection/commands.py +27 -2
  89. snowflake/cli/plugins/connection/plugin_spec.py +15 -1
  90. snowflake/cli/plugins/connection/util.py +21 -1
  91. snowflake/cli/plugins/cortex/__init__.py +13 -0
  92. snowflake/cli/plugins/cortex/commands.py +16 -2
  93. snowflake/cli/plugins/cortex/constants.py +14 -0
  94. snowflake/cli/plugins/cortex/manager.py +14 -0
  95. snowflake/cli/plugins/cortex/plugin_spec.py +15 -1
  96. snowflake/cli/plugins/cortex/types.py +14 -0
  97. snowflake/cli/plugins/git/__init__.py +13 -0
  98. snowflake/cli/plugins/git/commands.py +16 -2
  99. snowflake/cli/plugins/git/manager.py +14 -0
  100. snowflake/cli/plugins/git/plugin_spec.py +15 -1
  101. snowflake/cli/plugins/nativeapp/__init__.py +13 -0
  102. snowflake/cli/plugins/nativeapp/artifacts.py +202 -98
  103. snowflake/cli/plugins/nativeapp/codegen/__init__.py +13 -0
  104. snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py +14 -0
  105. snowflake/cli/plugins/nativeapp/codegen/compiler.py +14 -0
  106. snowflake/cli/plugins/nativeapp/codegen/sandbox.py +14 -0
  107. snowflake/cli/plugins/nativeapp/codegen/snowpark/callback_source.py.jinja +4 -4
  108. snowflake/cli/plugins/nativeapp/codegen/snowpark/extension_function_utils.py +34 -13
  109. snowflake/cli/plugins/nativeapp/codegen/snowpark/models.py +14 -0
  110. snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py +52 -13
  111. snowflake/cli/plugins/nativeapp/commands.py +63 -10
  112. snowflake/cli/plugins/nativeapp/common_flags.py +20 -0
  113. snowflake/cli/plugins/nativeapp/constants.py +15 -0
  114. snowflake/cli/plugins/nativeapp/exceptions.py +23 -2
  115. snowflake/cli/plugins/nativeapp/init.py +14 -0
  116. snowflake/cli/plugins/nativeapp/manager.py +118 -25
  117. snowflake/cli/plugins/nativeapp/plugin_spec.py +15 -1
  118. snowflake/cli/plugins/nativeapp/policy.py +14 -0
  119. snowflake/cli/plugins/nativeapp/run_processor.py +22 -3
  120. snowflake/cli/plugins/nativeapp/teardown_processor.py +14 -0
  121. snowflake/cli/plugins/nativeapp/utils.py +14 -0
  122. snowflake/cli/plugins/nativeapp/version/__init__.py +13 -0
  123. snowflake/cli/plugins/nativeapp/version/commands.py +19 -4
  124. snowflake/cli/plugins/nativeapp/version/version_processor.py +26 -4
  125. snowflake/cli/plugins/notebook/__init__.py +13 -0
  126. snowflake/cli/plugins/notebook/commands.py +16 -4
  127. snowflake/cli/plugins/notebook/exceptions.py +14 -0
  128. snowflake/cli/plugins/notebook/manager.py +14 -0
  129. snowflake/cli/plugins/notebook/plugin_spec.py +15 -1
  130. snowflake/cli/plugins/notebook/types.py +14 -0
  131. snowflake/cli/plugins/object/__init__.py +13 -11
  132. snowflake/cli/plugins/object/command_aliases.py +20 -6
  133. snowflake/cli/plugins/object/commands.py +16 -2
  134. snowflake/cli/plugins/object/common.py +14 -0
  135. snowflake/cli/plugins/object/manager.py +14 -0
  136. snowflake/cli/plugins/object/plugin_spec.py +16 -2
  137. snowflake/cli/plugins/object_stage_deprecated/__init__.py +14 -0
  138. snowflake/cli/plugins/object_stage_deprecated/commands.py +16 -2
  139. snowflake/cli/plugins/object_stage_deprecated/plugin_spec.py +16 -4
  140. snowflake/cli/plugins/snowpark/__init__.py +13 -4
  141. snowflake/cli/plugins/snowpark/commands.py +18 -2
  142. snowflake/cli/plugins/snowpark/common.py +15 -0
  143. snowflake/cli/plugins/snowpark/manager.py +14 -0
  144. snowflake/cli/plugins/snowpark/models.py +15 -1
  145. snowflake/cli/plugins/snowpark/package/__init__.py +13 -0
  146. snowflake/cli/plugins/snowpark/package/anaconda_packages.py +14 -0
  147. snowflake/cli/plugins/snowpark/package/commands.py +16 -2
  148. snowflake/cli/plugins/snowpark/package/manager.py +14 -0
  149. snowflake/cli/plugins/snowpark/package/utils.py +14 -0
  150. snowflake/cli/plugins/snowpark/package_utils.py +15 -1
  151. snowflake/cli/plugins/snowpark/plugin_spec.py +16 -2
  152. snowflake/cli/plugins/snowpark/snowpark_package_paths.py +14 -0
  153. snowflake/cli/plugins/snowpark/snowpark_shared.py +14 -0
  154. snowflake/cli/plugins/snowpark/zipper.py +20 -3
  155. snowflake/cli/plugins/spcs/__init__.py +16 -2
  156. snowflake/cli/plugins/spcs/common.py +14 -0
  157. snowflake/cli/plugins/spcs/compute_pool/__init__.py +13 -0
  158. snowflake/cli/plugins/spcs/compute_pool/commands.py +16 -2
  159. snowflake/cli/plugins/spcs/compute_pool/manager.py +14 -0
  160. snowflake/cli/plugins/spcs/image_registry/__init__.py +13 -0
  161. snowflake/cli/plugins/spcs/image_registry/commands.py +16 -2
  162. snowflake/cli/plugins/spcs/image_registry/manager.py +16 -0
  163. snowflake/cli/plugins/spcs/image_repository/__init__.py +13 -0
  164. snowflake/cli/plugins/spcs/image_repository/commands.py +16 -2
  165. snowflake/cli/plugins/spcs/image_repository/manager.py +14 -0
  166. snowflake/cli/plugins/spcs/jobs/__init__.py +13 -0
  167. snowflake/cli/plugins/spcs/jobs/commands.py +17 -3
  168. snowflake/cli/plugins/spcs/jobs/manager.py +14 -0
  169. snowflake/cli/plugins/spcs/plugin_spec.py +15 -1
  170. snowflake/cli/plugins/spcs/services/__init__.py +13 -0
  171. snowflake/cli/plugins/spcs/services/commands.py +16 -2
  172. snowflake/cli/plugins/spcs/services/manager.py +14 -0
  173. snowflake/cli/plugins/sql/__init__.py +13 -0
  174. snowflake/cli/plugins/sql/commands.py +16 -2
  175. snowflake/cli/plugins/sql/manager.py +14 -0
  176. snowflake/cli/plugins/sql/plugin_spec.py +15 -1
  177. snowflake/cli/plugins/sql/snowsql_templating.py +14 -0
  178. snowflake/cli/plugins/stage/__init__.py +13 -0
  179. snowflake/cli/plugins/stage/commands.py +17 -3
  180. snowflake/cli/plugins/stage/diff.py +14 -0
  181. snowflake/cli/plugins/stage/manager.py +16 -5
  182. snowflake/cli/plugins/stage/plugin_spec.py +16 -2
  183. snowflake/cli/plugins/streamlit/__init__.py +13 -0
  184. snowflake/cli/plugins/streamlit/commands.py +16 -2
  185. snowflake/cli/plugins/streamlit/manager.py +14 -0
  186. snowflake/cli/plugins/streamlit/plugin_spec.py +15 -1
  187. snowflake/cli/templates/default_snowpark/app/__init__.py +13 -0
  188. snowflake/cli/templates/default_snowpark/app/common.py +15 -0
  189. snowflake/cli/templates/default_snowpark/app/functions.py +14 -0
  190. snowflake/cli/templates/default_snowpark/app/procedures.py +14 -0
  191. snowflake/cli/templates/default_snowpark/snowflake.yml +1 -1
  192. snowflake/cli/templates/default_streamlit/common/hello.py +15 -0
  193. snowflake/cli/templates/default_streamlit/pages/my_page.py +14 -0
  194. snowflake/cli/templates/default_streamlit/snowflake.yml +1 -1
  195. snowflake/cli/templates/default_streamlit/streamlit_app.py +14 -0
  196. {snowflake_cli_labs-2.4.1.dist-info → snowflake_cli_labs-2.5.0rc0.dist-info}/METADATA +23 -7
  197. snowflake_cli_labs-2.5.0rc0.dist-info/RECORD +206 -0
  198. snowflake/cli/plugins/nativeapp/feature_flags.py +0 -10
  199. snowflake/cli/templates/environment.yml.jinja +0 -5
  200. snowflake/cli/templates/streamlit_app_launcher.py.jinja +0 -19
  201. snowflake_cli_labs-2.4.1.dist-info/RECORD +0 -206
  202. {snowflake_cli_labs-2.4.1.dist-info → snowflake_cli_labs-2.5.0rc0.dist-info}/WHEEL +0 -0
  203. {snowflake_cli_labs-2.4.1.dist-info → snowflake_cli_labs-2.5.0rc0.dist-info}/entry_points.txt +0 -0
  204. {snowflake_cli_labs-2.4.1.dist-info → snowflake_cli_labs-2.5.0rc0.dist-info}/licenses/LICENSE +0 -0
@@ -1,11 +1,24 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  from __future__ import annotations
2
16
 
3
17
  import itertools
4
18
  import os
5
- from dataclasses import dataclass
6
19
  from pathlib import Path
7
20
  from textwrap import dedent
8
- from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
21
+ from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
9
22
 
10
23
  from click.exceptions import ClickException
11
24
  from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB
@@ -43,7 +56,7 @@ class SourceNotFoundError(ClickException):
43
56
 
44
57
  class TooManyFilesError(ClickException):
45
58
  """
46
- Multiple files were mapped to one output file.
59
+ Multiple file or directories were mapped to one output destination.
47
60
  """
48
61
 
49
62
  dest_path: Path
@@ -85,17 +98,130 @@ class NotInDeployRootError(ClickException):
85
98
  self.src_path = src_path
86
99
 
87
100
 
88
- @dataclass
89
- class ArtifactMapping:
101
+ ArtifactPredicate = Callable[[Path, Path], bool]
102
+
103
+
104
+ class _ArtifactPathMap:
90
105
  """
91
- Used to keep track of equivalent paths / globs so we can copy artifacts from the project folder to the deploy root.
106
+ A specialized version of an ordered multimap used to keep track of artifact
107
+ source-destination mappings. The mapping is bidirectional, so it can be queried
108
+ by source or destination paths. All paths manipulated by this class must be in
109
+ relative, canonical form (relative to the project or deploy roots, as appropriate).
92
110
  """
93
111
 
94
- src: str
95
- dest: str
112
+ def __init__(self, project_root: Path):
113
+ self._project_root = project_root
96
114
 
115
+ # All (src,dest) pairs in inserting order, for iterating
116
+ self.__src_dest_pairs: List[Tuple[Path, Path]] = []
117
+ # built-in dict instances are ordered as of Python 3.7
118
+ self.__src_to_dest: Dict[Path, List[Path]] = {}
119
+ self.__dest_to_src: Dict[Path, List[Path]] = {}
97
120
 
98
- ArtifactPredicate = Callable[[Path, Path], bool]
121
+ # This dictionary accumulates keys for each directory or file to be created in
122
+ # the deploy root for any artifact mapping rule being processed. This includes
123
+ # children of directories that are copied to the deploy root. Having this
124
+ # information available is critical to detect possible clashes between rules.
125
+ self._dest_is_dir: Dict[Path, bool] = {}
126
+
127
+ def put(self, src: Path, dest: Path, dest_is_dir: bool) -> None:
128
+ """
129
+ Adds a new source-destination mapping pair to this map, if necessary. Note that
130
+ this is internal logic that assumes that src-dest pairs have already been preprocessed
131
+ by the enclosing BundleMap (for example, only file -> file and
132
+ directory -> directory mappings are possible here due to the preprocessing step).
133
+
134
+ Arguments:
135
+ src {Path} -- the source path, in canonical form.
136
+ dest {Path} -- the destination path, in canonical form.
137
+ dest_is_dir {bool} -- whether the destination path is a directory.
138
+ """
139
+ # Both paths should be in canonical form
140
+ assert not src.is_absolute()
141
+ assert not dest.is_absolute()
142
+
143
+ absolute_src = self._project_root / src
144
+
145
+ current_sources = self.__dest_to_src.get(dest, [])
146
+ src_is_dir = absolute_src.is_dir()
147
+ if dest_is_dir:
148
+ assert src_is_dir # file -> directory is not possible here given how rules are processed
149
+
150
+ # directory -> directory
151
+ # Check that dest is currently unmapped
152
+ current_is_dir = self._dest_is_dir.get(dest, False)
153
+ if current_is_dir:
154
+ # mapping to an existing directory is not allowed
155
+ raise TooManyFilesError(dest)
156
+ else:
157
+ # file -> file
158
+ # Check that there is no previous mapping for the same file.
159
+ if current_sources and src not in current_sources:
160
+ # There is already a different source mapping to this destination
161
+ raise TooManyFilesError(dest)
162
+
163
+ if src_is_dir:
164
+ # mark all subdirectories of this source as directories so that we can
165
+ # detect accidental clobbering
166
+ for (root, _, files) in os.walk(absolute_src, followlinks=True):
167
+ canonical_subdir = Path(root).relative_to(absolute_src)
168
+ canonical_dest_subdir = dest / canonical_subdir
169
+ self._update_dest_is_dir(canonical_dest_subdir, is_dir=True)
170
+ for f in files:
171
+ self._update_dest_is_dir(canonical_dest_subdir / f, is_dir=False)
172
+
173
+ # make sure we check for dest_is_dir consistency regardless of whether the
174
+ # insertion happened. This update can fail, so we need to do it first to
175
+ # avoid applying partial updates to the underlying data storage.
176
+ self._update_dest_is_dir(dest, dest_is_dir)
177
+
178
+ dests = self.__src_to_dest.setdefault(src, [])
179
+ srcs = self.__dest_to_src.setdefault(dest, [])
180
+ if dest not in dests:
181
+ dests.append(dest)
182
+ srcs.append(src)
183
+ self.__src_dest_pairs.append((src, dest))
184
+
185
+ def get_sources(self, dest: Path) -> Iterable[Path]:
186
+ """
187
+ Returns all source paths associated with the provided destination path, in insertion order.
188
+ """
189
+ return self.__dest_to_src.get(dest, [])
190
+
191
+ def get_destinations(self, src: Path) -> Iterable[Path]:
192
+ """
193
+ Returns all destination paths associated with the provided source path, in insertion order.
194
+ """
195
+ return self.__src_to_dest.get(src, [])
196
+
197
+ def __iter__(self) -> Iterator[Tuple[Path, Path]]:
198
+ """
199
+ Returns all (source, destination) pairs known to this map, in insertion order.
200
+ """
201
+ return iter(self.__src_dest_pairs)
202
+
203
+ def _update_dest_is_dir(self, dest: Path, is_dir: bool) -> None:
204
+ """
205
+ Recursively marks seen destination paths as either files or folders, raising an error if any inconsistencies
206
+ from previous invocations of this method are encountered.
207
+
208
+ Arguments:
209
+ dest {Path} -- the destination path, in canonical form.
210
+ is_dir {bool} -- whether the destination path is a directory.
211
+ """
212
+ assert not dest.is_absolute() # dest must be in canonical relative form
213
+
214
+ current_is_dir = self._dest_is_dir.get(dest, None)
215
+ if current_is_dir is not None and current_is_dir != is_dir:
216
+ raise ArtifactError(
217
+ "Conflicting type for destination path: {canonical_dest}"
218
+ )
219
+
220
+ parent = dest.parent
221
+ if parent != dest:
222
+ self._update_dest_is_dir(parent, True)
223
+
224
+ self._dest_is_dir[dest] = is_dir
99
225
 
100
226
 
101
227
  class BundleMap:
@@ -107,9 +233,7 @@ class BundleMap:
107
233
  def __init__(self, *, project_root: Path, deploy_root: Path):
108
234
  self._project_root: Path = resolve_without_follow(project_root)
109
235
  self._deploy_root: Path = resolve_without_follow(deploy_root)
110
- self._src_to_dest: Dict[Path, List[Path]] = {}
111
- self._dest_to_src: Dict[Path, List[Path]] = {}
112
- self._dest_is_dir: Dict[Path, bool] = {}
236
+ self._artifact_map = _ArtifactPathMap(project_root=self._project_root)
113
237
 
114
238
  def deploy_root(self) -> Path:
115
239
  return self._deploy_root
@@ -137,7 +261,7 @@ class BundleMap:
137
261
  )
138
262
 
139
263
  if self._deploy_root in absolute_src.parents:
140
- # ignore this item since it's in the deploy root. This can happen if the bundle map is create
264
+ # ignore this item since it's in the deploy root. This can happen if the bundle map is created
141
265
  # after the bundle step and a project is using rules that are not sufficiently constrained.
142
266
  # Since the bundle step starts with deleting the deploy root, we wouldn't normally encounter this situation.
143
267
  return
@@ -145,28 +269,16 @@ class BundleMap:
145
269
  canonical_src = self._canonical_src(src)
146
270
  canonical_dest = self._canonical_dest(dest)
147
271
 
148
- src_is_dir = absolute_src.is_dir()
149
272
  if map_as_child:
150
273
  # Make sure the destination is a child of the original, since this was requested
151
274
  canonical_dest = canonical_dest / canonical_src.name
152
- dest_is_dir = src.is_dir()
153
-
154
- # Verify that multiple files are not being mapped to a single file destination
155
- current_sources = self._dest_to_src.setdefault(canonical_dest, [])
156
- if not dest_is_dir:
157
- # the destination is a file
158
- if (canonical_src not in current_sources) and len(current_sources) > 0:
159
- raise TooManyFilesError(dest)
275
+ dest_is_dir = absolute_src.is_dir()
160
276
 
161
- # Perform all updates together we don't end up with inconsistent state
162
- self._update_dest_is_dir(canonical_dest, dest_is_dir)
163
- current_dests = self._src_to_dest.setdefault(canonical_src, [])
164
- if canonical_dest not in current_dests:
165
- current_dests.append(canonical_dest)
166
- if canonical_src not in current_sources:
167
- current_sources.append(canonical_src)
277
+ self._artifact_map.put(
278
+ src=canonical_src, dest=canonical_dest, dest_is_dir=dest_is_dir
279
+ )
168
280
 
169
- def _add_mapping(self, src: str, dest: Optional[str]):
281
+ def _add_mapping(self, src: str, dest: Optional[str] = None):
170
282
  """
171
283
  Adds the specified artifact rule to this instance. The source should be relative to the project directory. It
172
284
  is interpreted as a file, directory or glob pattern. If the destination path is not specified, each source match
@@ -206,35 +318,42 @@ class BundleMap:
206
318
  if not match_found:
207
319
  raise SourceNotFoundError(src)
208
320
 
209
- def add(self, mapping: Union[ArtifactMapping, PathMapping]) -> None:
321
+ def add(self, mapping: PathMapping) -> None:
210
322
  """
211
323
  Adds an artifact mapping rule to this instance.
212
324
  """
213
- if isinstance(mapping, ArtifactMapping):
214
- self._add_mapping(mapping.src, mapping.dest)
215
- elif isinstance(mapping, PathMapping):
216
- self._add_mapping(mapping.src, mapping.dest)
217
- else:
218
- raise RuntimeError(f"Unsupported mapping type: {type(mapping)}")
325
+ self._add_mapping(mapping.src, mapping.dest)
219
326
 
220
- def _mappings_for_source(
327
+ def _expand_artifact_mapping(
221
328
  self,
222
329
  src: Path,
330
+ dest: Path,
223
331
  absolute: bool = False,
224
332
  expand_directories: bool = False,
225
333
  predicate: ArtifactPredicate = lambda src, dest: True,
226
334
  ) -> Iterator[Tuple[Path, Path]]:
335
+ """
336
+ Expands the specified source-destination mapping according to the provided options.
337
+ The original mapping is yielded, followed by any expanded mappings derived from
338
+ it.
339
+
340
+ Arguments:
341
+ src {Path} -- the source path
342
+ dest {Path} -- the destination path
343
+ absolute {bool} -- when True, all mappings will be yielded as absolute paths
344
+ expand_directories {bool} -- when True, child mappings are yielded if the source path is a directory.
345
+ predicate {ArtifactPredicate} -- when specified, only mappings satisfying this predicate will be yielded.
346
+ """
227
347
  canonical_src = self._canonical_src(src)
228
- canonical_dests = self._src_to_dest.get(canonical_src)
229
- assert canonical_dests is not None
348
+ canonical_dest = self._canonical_dest(dest)
230
349
 
231
350
  absolute_src = self._absolute_src(canonical_src)
351
+ absolute_dest = self._absolute_dest(canonical_dest)
232
352
  src_for_output = self._to_output_src(absolute_src, absolute)
233
- dests_for_output = [self._to_output_dest(p, absolute) for p in canonical_dests]
353
+ dest_for_output = self._to_output_dest(absolute_dest, absolute)
234
354
 
235
- for d in dests_for_output:
236
- if predicate(src_for_output, d):
237
- yield src_for_output, d
355
+ if predicate(src_for_output, dest_for_output):
356
+ yield src_for_output, dest_for_output
238
357
 
239
358
  if absolute_src.is_dir() and expand_directories:
240
359
  # both src and dest are directories, and expanding directories was requested. Traverse src, and map each
@@ -242,11 +361,10 @@ class BundleMap:
242
361
  for (root, subdirs, files) in os.walk(absolute_src, followlinks=True):
243
362
  relative_root = Path(root).relative_to(absolute_src)
244
363
  for name in itertools.chain(subdirs, files):
245
- for d in dests_for_output:
246
- src_file_for_output = src_for_output / relative_root / name
247
- dest_file_for_output = d / relative_root / name
248
- if predicate(src_file_for_output, dest_file_for_output):
249
- yield src_file_for_output, dest_file_for_output
364
+ src_file_for_output = src_for_output / relative_root / name
365
+ dest_file_for_output = dest_for_output / relative_root / name
366
+ if predicate(src_file_for_output, dest_file_for_output):
367
+ yield src_file_for_output, dest_file_for_output
250
368
 
251
369
  def all_mappings(
252
370
  self,
@@ -270,9 +388,10 @@ class BundleMap:
270
388
  Returns:
271
389
  An iterator over all matching deployed artifacts.
272
390
  """
273
- for src in self._src_to_dest.keys():
274
- for deployed_src, deployed_dest in self._mappings_for_source(
391
+ for src, dest in self._artifact_map:
392
+ for deployed_src, deployed_dest in self._expand_artifact_mapping(
275
393
  src,
394
+ dest,
276
395
  absolute=absolute,
277
396
  expand_directories=expand_directories,
278
397
  predicate=predicate,
@@ -300,8 +419,8 @@ class BundleMap:
300
419
 
301
420
  output_destinations: List[Path] = []
302
421
 
303
- canonical_dests = self._src_to_dest.get(canonical_src)
304
- if canonical_dests is not None:
422
+ canonical_dests = self._artifact_map.get_destinations(canonical_src)
423
+ if canonical_dests:
305
424
  for d in canonical_dests:
306
425
  output_destinations.append(self._to_output_dest(d, is_absolute))
307
426
 
@@ -362,19 +481,6 @@ class BundleMap:
362
481
  def _to_output_src(self, src: Path, absolute: bool) -> Path:
363
482
  return self._absolute_src(src) if absolute else self._canonical_src(src)
364
483
 
365
- def _update_dest_is_dir(self, canonical_dest: Path, is_dir: bool) -> None:
366
- current_is_dir = self._dest_is_dir.get(canonical_dest, None)
367
- if current_is_dir is not None and is_dir != current_is_dir:
368
- raise ArtifactError(
369
- "Conflicting type for destination path: {canonical_dest}"
370
- )
371
-
372
- parent = canonical_dest.parent
373
- if parent != canonical_dest:
374
- self._update_dest_is_dir(parent, True)
375
-
376
- self._dest_is_dir[canonical_dest] = is_dir
377
-
378
484
 
379
485
  def specifies_directory(s: str) -> bool:
380
486
  """
@@ -402,49 +508,47 @@ def delete(path: Path) -> None:
402
508
  spath.rmdir(recursive=True) # remove dir and all contains
403
509
 
404
510
 
405
- def symlink_or_copy(
406
- src: Path, dst: Path, deploy_root: Path, makedirs=True, overwrite=True
407
- ) -> None:
511
+ def symlink_or_copy(src: Path, dst: Path, deploy_root: Path) -> None:
408
512
  """
409
- Tries to create a symlink to src at dst; failing that (i.e. in Windows
410
- without Administrator / Developer Mode) copies the file from src to dst instead.
411
- If makedirs is True, the directory hierarchy above dst is created if any
412
- of those directories do not exist.
513
+ Symlinks files from src to dst. If the src contains parent directories, then copies the empty directory shell to the deploy root.
514
+ The directory hierarchy above dst is created if any of those directories do not exist.
413
515
  """
414
516
  ssrc = SecurePath(src)
415
517
  sdst = SecurePath(dst)
416
- if makedirs:
417
- sdst.parent.mkdir(parents=True, exist_ok=True)
518
+ sdst.parent.mkdir(parents=True, exist_ok=True)
418
519
 
419
520
  # Verify that the mapping isn't accidentally trying to create a file in the project source through symlinks.
420
521
  # We need to ensure we're resolving symlinks for this check to be effective.
522
+ # We are unlikely to hit this if calling the function through bundle map, keeping it here for other future use cases outside bundle.
421
523
  resolved_dst = dst.resolve()
422
524
  resolved_deploy_root = deploy_root.resolve()
423
- if resolved_deploy_root not in resolved_dst.parents:
525
+ dst_is_deploy_root = resolved_deploy_root == resolved_dst
526
+ if (not dst_is_deploy_root) and (resolved_deploy_root not in resolved_dst.parents):
424
527
  raise NotInDeployRootError(dest_path=dst, deploy_root=deploy_root, src_path=src)
425
528
 
426
- if overwrite:
529
+ absolute_src = resolve_without_follow(src)
530
+ if absolute_src.is_file():
427
531
  delete(dst)
428
- try:
429
- os.symlink(src, dst)
430
- except OSError:
431
- ssrc.copy(dst)
432
-
433
-
434
- def translate_artifact(item: Union[PathMapping, str]) -> ArtifactMapping:
435
- """
436
- Builds an artifact mapping from a project definition value.
437
- Validation is done later when we actually resolve files / folders.
438
- """
439
-
440
- if isinstance(item, PathMapping):
441
- return ArtifactMapping(item.src, item.dest if item.dest else item.src)
442
-
443
- elif isinstance(item, str):
444
- return ArtifactMapping(item, item)
445
-
446
- # XXX: validation should have caught this
447
- raise ArtifactError("Item is not a valid artifact!")
532
+ try:
533
+ os.symlink(absolute_src, dst)
534
+ except OSError:
535
+ ssrc.copy(dst)
536
+ else:
537
+ # 1. Create a new directory in the deploy root
538
+ dst.mkdir(exist_ok=True)
539
+ # 2. For all children of src, create their counterparts in dst now that it exists
540
+ for root, _, files in sorted(os.walk(absolute_src, followlinks=True)):
541
+ relative_root = Path(root).relative_to(absolute_src)
542
+ absolute_root_in_deploy = Path(dst, relative_root)
543
+ absolute_root_in_deploy.mkdir(parents=True, exist_ok=True)
544
+ for file in files:
545
+ absolute_file_in_project = Path(absolute_src, relative_root, file)
546
+ absolute_file_in_deploy = Path(absolute_root_in_deploy, file)
547
+ symlink_or_copy(
548
+ src=absolute_file_in_project,
549
+ dst=absolute_file_in_deploy,
550
+ deploy_root=deploy_root,
551
+ )
448
552
 
449
553
 
450
554
  def resolve_without_follow(path: Path) -> Path:
@@ -458,7 +562,7 @@ def resolve_without_follow(path: Path) -> Path:
458
562
  def build_bundle(
459
563
  project_root: Path,
460
564
  deploy_root: Path,
461
- artifacts: List[ArtifactMapping],
565
+ artifacts: List[PathMapping],
462
566
  ) -> BundleMap:
463
567
  """
464
568
  Prepares a local folder (deploy_root) with configured app artifacts.
@@ -0,0 +1,13 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
@@ -1,3 +1,17 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  from __future__ import annotations
2
16
 
3
17
  from abc import ABC, abstractmethod
@@ -1,3 +1,17 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  from __future__ import annotations
2
16
 
3
17
  from pathlib import Path
@@ -1,3 +1,17 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  from __future__ import annotations
2
16
 
3
17
  import os
@@ -2,7 +2,7 @@ import contextlib
2
2
  import functools
3
3
  import inspect
4
4
  import sys
5
- from typing import Callable
5
+ from typing import Callable, Tuple
6
6
 
7
7
  try:
8
8
  import snowflake.snowpark
@@ -101,8 +101,7 @@ def __snowflake_internal_create_extension_fn_registration_callback():
101
101
 
102
102
 
103
103
  def __snowflake_internal_extension_fn_to_json(extension_fn):
104
- if not isinstance(extension_fn.func, Callable):
105
- # Unsupported case: extension function is a tuple
104
+ if not (isinstance(extension_fn.func, Callable) or isinstance(extension_fn.func, Tuple)):
106
105
  return
107
106
 
108
107
  if extension_fn.anonymous:
@@ -141,7 +140,8 @@ def __snowflake_internal_create_extension_fn_registration_callback():
141
140
  collected_extension_fn_json_list, extension_function_properties
142
141
  ):
143
142
  extension_fn_json = __snowflake_internal_extension_fn_to_json(extension_function_properties)
144
- collected_extension_fn_json_list.append(extension_fn_json)
143
+ if extension_fn_json: # Do not append if extension_fn_json is None
144
+ collected_extension_fn_json_list.append(extension_fn_json)
145
145
  return False
146
146
 
147
147
  return functools.partial(
@@ -1,11 +1,25 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  from __future__ import annotations
2
16
 
3
17
  import ast
4
18
  from typing import (
5
- Any,
6
19
  List,
7
20
  Optional,
8
21
  Sequence,
22
+ Union,
9
23
  )
10
24
 
11
25
  from click.exceptions import ClickException
@@ -21,6 +35,8 @@ from snowflake.cli.plugins.nativeapp.codegen.snowpark.models import (
21
35
  NativeAppExtensionFunction,
22
36
  )
23
37
 
38
+ ASTDefNode = Union[ast.FunctionDef, ast.ClassDef]
39
+
24
40
 
25
41
  class MalformedExtensionFunctionError(ClickException):
26
42
  """Required extension function attribute is missing."""
@@ -96,30 +112,35 @@ def ensure_all_string_literals(values: Sequence[str]) -> List[str]:
96
112
  return [ensure_string_literal(value) for value in values]
97
113
 
98
114
 
99
- class _FunctionDefAccumulator(ast.NodeVisitor):
115
+ class _SnowparkHandlerAccumulator(ast.NodeVisitor):
100
116
  """
101
- A NodeVisitor that collects AST nodes corresponding to function declarations, filtered by a list
102
- of wanted functions. This is used to identify all Snowpark extension functions in a module's
103
- source code.
117
+ A NodeVisitor that collects AST nodes corresponding to a provided list of Snowpark external functions.
118
+ The returned nodes are filtered using the handlers provided for each of the Snowpark functions.
119
+ Returned definitions can be either function definition or class definition AST nodes.
104
120
  """
105
121
 
106
122
  def __init__(self, functions: Sequence[NativeAppExtensionFunction]):
107
- self._wanted_functions_by_name = {
123
+ self._wanted_handlers_by_name = {
108
124
  fn.handler.split(".")[-1]: fn for fn in functions
109
125
  }
110
- self.definitions: List[Any] = []
126
+ self.definitions: List[ASTDefNode] = []
111
127
 
112
128
  def visit_FunctionDef(self, node: ast.FunctionDef): # noqa: N802
113
129
  if self._want(node):
114
130
  self.definitions.append(node)
115
131
  self.generic_visit(node)
116
132
 
117
- def _want(self, node: Any) -> bool:
133
+ def visit_ClassDef(self, node: ast.ClassDef): # noqa: N802
134
+ if self._want(node):
135
+ self.definitions.append(node)
136
+ self.generic_visit(node)
137
+
138
+ def _want(self, node: ASTDefNode) -> bool:
118
139
  if not node.decorator_list:
119
140
  # No decorators for this definition, ignore it
120
141
  return False
121
142
 
122
- return node.name in self._wanted_functions_by_name
143
+ return node.name in self._wanted_handlers_by_name
123
144
 
124
145
 
125
146
  def _get_decorator_id(node: ast.AST) -> Optional[str]:
@@ -136,10 +157,10 @@ def _get_decorator_id(node: ast.AST) -> Optional[str]:
136
157
  return None
137
158
 
138
159
 
139
- def _collect_ast_function_definitions(
160
+ def _collect_ast_handler_definitions(
140
161
  tree: ast.AST, extension_functions: Sequence[NativeAppExtensionFunction]
141
- ) -> Sequence[ast.FunctionDef]:
142
- accumulator = _FunctionDefAccumulator(extension_functions)
162
+ ) -> Sequence[ASTDefNode]:
163
+ accumulator = _SnowparkHandlerAccumulator(extension_functions)
143
164
  accumulator.visit(tree)
144
165
  return accumulator.definitions
145
166
 
@@ -167,7 +188,7 @@ def deannotate_module_source(
167
188
 
168
189
  tree = ast.parse(module_source)
169
190
 
170
- definitions = _collect_ast_function_definitions(tree, extension_functions)
191
+ definitions = _collect_ast_handler_definitions(tree, extension_functions)
171
192
  if not definitions:
172
193
  return module_source
173
194
 
@@ -1,3 +1,17 @@
1
+ # Copyright (c) 2024 Snowflake Inc.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
1
15
  from __future__ import annotations
2
16
 
3
17
  from enum import Enum