rsconnect-python 1.30.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rsconnect/__init__.py +13 -0
- rsconnect/actions.py +565 -0
- rsconnect/actions_content.py +508 -0
- rsconnect/actions_environment.py +160 -0
- rsconnect/actions_integration.py +118 -0
- rsconnect/api.py +2582 -0
- rsconnect/bundle.py +2481 -0
- rsconnect/certificates.py +39 -0
- rsconnect/environment.py +390 -0
- rsconnect/environment_node.py +115 -0
- rsconnect/environment_r.py +300 -0
- rsconnect/exception.py +15 -0
- rsconnect/git_metadata.py +180 -0
- rsconnect/http_support.py +595 -0
- rsconnect/json_web_token.py +178 -0
- rsconnect/log.py +253 -0
- rsconnect/main.py +5889 -0
- rsconnect/metadata.py +879 -0
- rsconnect/models.py +835 -0
- rsconnect/oauth.py +623 -0
- rsconnect/py.typed +0 -0
- rsconnect/pyproject.py +283 -0
- rsconnect/quickstart/__init__.py +16 -0
- rsconnect/quickstart/quickstart.py +486 -0
- rsconnect/quickstart/templates/__init__.py +16 -0
- rsconnect/quickstart/templates/api/README.md.tmpl +15 -0
- rsconnect/quickstart/templates/api/__connect__.py.tmpl +3 -0
- rsconnect/quickstart/templates/api/__init__.py.tmpl +1 -0
- rsconnect/quickstart/templates/api/__main__.py.tmpl +14 -0
- rsconnect/quickstart/templates/api/app.py.tmpl +11 -0
- rsconnect/quickstart/templates/api/pyproject.toml.tmpl +13 -0
- rsconnect/quickstart/templates/fastapi/README.md.tmpl +15 -0
- rsconnect/quickstart/templates/fastapi/__connect__.py.tmpl +3 -0
- rsconnect/quickstart/templates/fastapi/__init__.py.tmpl +1 -0
- rsconnect/quickstart/templates/fastapi/__main__.py.tmpl +16 -0
- rsconnect/quickstart/templates/fastapi/app.py.tmpl +11 -0
- rsconnect/quickstart/templates/fastapi/pyproject.toml.tmpl +14 -0
- rsconnect/quickstart/templates/notebook/README.md.tmpl +15 -0
- rsconnect/quickstart/templates/notebook/notebook.ipynb.tmpl +34 -0
- rsconnect/quickstart/templates/notebook/pyproject.toml.tmpl +13 -0
- rsconnect/quickstart/templates/quarto/README.md.tmpl +19 -0
- rsconnect/quickstart/templates/quarto/pyproject.toml.tmpl +11 -0
- rsconnect/quickstart/templates/quarto/report.qmd.tmpl +8 -0
- rsconnect/quickstart/templates/shiny/README.md.tmpl +15 -0
- rsconnect/quickstart/templates/shiny/app.py.tmpl +3 -0
- rsconnect/quickstart/templates/shiny/pyproject.toml.tmpl +13 -0
- rsconnect/quickstart/templates/streamlit/README.md.tmpl +15 -0
- rsconnect/quickstart/templates/streamlit/app.py.tmpl +3 -0
- rsconnect/quickstart/templates/streamlit/pyproject.toml.tmpl +13 -0
- rsconnect/quickstart/templates/voila/README.md.tmpl +15 -0
- rsconnect/quickstart/templates/voila/pyproject.toml.tmpl +14 -0
- rsconnect/shiny_express.py +136 -0
- rsconnect/snowflake.py +93 -0
- rsconnect/subprocesses/__init__.py +0 -0
- rsconnect/subprocesses/inspect_environment.py +362 -0
- rsconnect/timeouts.py +89 -0
- rsconnect/utils_package.py +261 -0
- rsconnect/validation.py +156 -0
- rsconnect/version_check.py +154 -0
- rsconnect_python-1.30.0.dist-info/METADATA +89 -0
- rsconnect_python-1.30.0.dist-info/RECORD +63 -0
- rsconnect_python-1.30.0.dist-info/WHEEL +4 -0
- rsconnect_python-1.30.0.dist-info/entry_points.txt +3 -0
rsconnect/bundle.py
ADDED
|
@@ -0,0 +1,2481 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Manifest generation and bundling utilities
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import hashlib
|
|
8
|
+
import io
|
|
9
|
+
import json
|
|
10
|
+
import mimetypes
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tarfile
|
|
16
|
+
import tempfile
|
|
17
|
+
import typing
|
|
18
|
+
from collections import defaultdict
|
|
19
|
+
from copy import deepcopy
|
|
20
|
+
from mimetypes import guess_type
|
|
21
|
+
from os.path import (
|
|
22
|
+
abspath,
|
|
23
|
+
basename,
|
|
24
|
+
dirname,
|
|
25
|
+
exists,
|
|
26
|
+
isdir,
|
|
27
|
+
isfile,
|
|
28
|
+
join,
|
|
29
|
+
relpath,
|
|
30
|
+
splitext,
|
|
31
|
+
)
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
from typing import (
|
|
34
|
+
IO,
|
|
35
|
+
TYPE_CHECKING,
|
|
36
|
+
Callable,
|
|
37
|
+
Iterator,
|
|
38
|
+
Literal,
|
|
39
|
+
Optional,
|
|
40
|
+
Sequence,
|
|
41
|
+
Union,
|
|
42
|
+
cast,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
# Even though TypedDict is available in Python 3.8, because it's used with NotRequired,
|
|
46
|
+
# they should both come from the same typing module.
|
|
47
|
+
# https://peps.python.org/pep-0655/#usage-in-python-3-11
|
|
48
|
+
if sys.version_info >= (3, 11):
|
|
49
|
+
from typing import NotRequired, TypedDict
|
|
50
|
+
else:
|
|
51
|
+
from typing_extensions import NotRequired, TypedDict
|
|
52
|
+
|
|
53
|
+
import click
|
|
54
|
+
|
|
55
|
+
from .environment import Environment, list_environment_dirs, is_environment_dir
|
|
56
|
+
from .environment_node import NodeEnvironment
|
|
57
|
+
from .environment_r import REnvironment
|
|
58
|
+
from .exception import RSConnectException
|
|
59
|
+
from .log import VERBOSE, logger
|
|
60
|
+
from .models import AppMode, AppModes, GlobSet
|
|
61
|
+
from .shiny_express import escape_to_var_name, is_express_app
|
|
62
|
+
|
|
63
|
+
if TYPE_CHECKING:
|
|
64
|
+
from .actions import QuartoInspectResult
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_module_pattern = re.compile(r"^[A-Za-z0-9_]+:[A-Za-z0-9_]+$")
|
|
68
|
+
|
|
69
|
+
# From https://github.com/rstudio/rsconnect/blob/485e05a26041ab8183a220da7a506c9d3a41f1ff/R/bundle.R#L85-L88
|
|
70
|
+
# noinspection SpellCheckingInspection
|
|
71
|
+
directories_ignore_list = [
|
|
72
|
+
".Rproj.user/",
|
|
73
|
+
".git/",
|
|
74
|
+
".svn/",
|
|
75
|
+
"__pycache__/",
|
|
76
|
+
"packrat/",
|
|
77
|
+
"renv/",
|
|
78
|
+
"rsconnect-python/",
|
|
79
|
+
"rsconnect/",
|
|
80
|
+
"node_modules/",
|
|
81
|
+
]
|
|
82
|
+
directories_to_ignore = {Path(d) for d in directories_ignore_list}
|
|
83
|
+
|
|
84
|
+
mimetypes.add_type("text/ipynb", ".ipynb")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class ManifestDataFile(TypedDict):
|
|
88
|
+
checksum: str
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class ManifestDataMetadata(TypedDict):
|
|
92
|
+
appmode: str
|
|
93
|
+
primary_html: NotRequired[str]
|
|
94
|
+
entrypoint: NotRequired[str]
|
|
95
|
+
primary_rmd: NotRequired[str]
|
|
96
|
+
content_category: NotRequired[str]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class ManifestDataJupyter(TypedDict):
|
|
100
|
+
hide_all_input: NotRequired[bool]
|
|
101
|
+
hide_tagged_input: NotRequired[bool]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class ManifestDataQuarto(TypedDict):
|
|
105
|
+
version: str
|
|
106
|
+
engines: list[str]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class ManifestDataEnvironmentPython(TypedDict):
|
|
110
|
+
requires: NotRequired[str]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ManifestDataEnvironment(TypedDict):
|
|
114
|
+
image: NotRequired[str]
|
|
115
|
+
environment_management: NotRequired[dict[Literal["python", "r", "node"], bool]]
|
|
116
|
+
python: NotRequired[ManifestDataEnvironmentPython]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class ManifestDataPython(TypedDict):
|
|
120
|
+
version: str
|
|
121
|
+
package_manager: ManifestDataPythonPackageManager
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class ManifestDataPythonPackageManager(TypedDict):
|
|
125
|
+
name: str
|
|
126
|
+
version: str
|
|
127
|
+
package_file: str
|
|
128
|
+
# When set, hints server how to perform installs.
|
|
129
|
+
# If True, server may perform installs using `uv`.
|
|
130
|
+
# If False, server should not use `uv`.
|
|
131
|
+
# If omitted, behavior is server-driven (migration default).
|
|
132
|
+
allow_uv: NotRequired[bool]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class ManifestDataNodePackageManager(TypedDict):
|
|
136
|
+
name: str
|
|
137
|
+
version: str
|
|
138
|
+
package_file: str
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class ManifestDataNode(TypedDict):
|
|
142
|
+
version: str
|
|
143
|
+
package_manager: ManifestDataNodePackageManager
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class ManifestDataRPackage(TypedDict):
|
|
147
|
+
Source: str
|
|
148
|
+
Repository: str
|
|
149
|
+
description: dict[str, str]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class ManifestData(TypedDict):
|
|
153
|
+
version: int
|
|
154
|
+
files: dict[str, ManifestDataFile]
|
|
155
|
+
locale: NotRequired[str]
|
|
156
|
+
metadata: ManifestDataMetadata
|
|
157
|
+
jupyter: NotRequired[ManifestDataJupyter]
|
|
158
|
+
quarto: NotRequired[ManifestDataQuarto]
|
|
159
|
+
python: NotRequired[ManifestDataPython]
|
|
160
|
+
node: NotRequired[ManifestDataNode]
|
|
161
|
+
platform: NotRequired[str]
|
|
162
|
+
packages: NotRequired[dict[str, ManifestDataRPackage]]
|
|
163
|
+
environment: NotRequired[ManifestDataEnvironment]
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class Manifest:
|
|
167
|
+
def __init__(
|
|
168
|
+
self,
|
|
169
|
+
version: Optional[int] = None,
|
|
170
|
+
environment: Optional[Environment] = None,
|
|
171
|
+
r_environment: Optional[REnvironment] = None,
|
|
172
|
+
app_mode: Optional[AppMode] = None,
|
|
173
|
+
entrypoint: Optional[str] = None,
|
|
174
|
+
quarto_inspection: Optional[QuartoInspectResult] = None,
|
|
175
|
+
image: Optional[str] = None,
|
|
176
|
+
env_management_py: Optional[bool] = None,
|
|
177
|
+
env_management_r: Optional[bool] = None,
|
|
178
|
+
primary_html: Optional[str] = None,
|
|
179
|
+
metadata: Optional[ManifestDataMetadata] = None,
|
|
180
|
+
files: Optional[dict[str, ManifestDataFile]] = None,
|
|
181
|
+
) -> None:
|
|
182
|
+
self.data: ManifestData = cast(ManifestData, {})
|
|
183
|
+
self.buffer: dict[str, str] = {}
|
|
184
|
+
self.deploy_dir: str | None = None
|
|
185
|
+
|
|
186
|
+
self.data["version"] = version if version else 1
|
|
187
|
+
if environment and environment.locale is not None:
|
|
188
|
+
self.data["locale"] = environment.locale
|
|
189
|
+
|
|
190
|
+
if metadata is None:
|
|
191
|
+
self.data["metadata"] = cast(ManifestDataMetadata, {})
|
|
192
|
+
if app_mode is None:
|
|
193
|
+
self.data["metadata"]["appmode"] = AppModes.UNKNOWN.name()
|
|
194
|
+
else:
|
|
195
|
+
self.data["metadata"]["appmode"] = app_mode.name()
|
|
196
|
+
else:
|
|
197
|
+
self.data["metadata"] = metadata
|
|
198
|
+
|
|
199
|
+
if primary_html:
|
|
200
|
+
self.data["metadata"]["primary_html"] = primary_html
|
|
201
|
+
|
|
202
|
+
if entrypoint:
|
|
203
|
+
self.data["metadata"]["entrypoint"] = entrypoint
|
|
204
|
+
|
|
205
|
+
if quarto_inspection:
|
|
206
|
+
self.data["quarto"] = {
|
|
207
|
+
"version": quarto_inspection.get("quarto", {}).get("version", "99.9.9"),
|
|
208
|
+
"engines": quarto_inspection.get("engines", []),
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
files_data = quarto_inspection.get("files", {})
|
|
212
|
+
files_input_data = files_data.get("input", [])
|
|
213
|
+
if len(files_input_data) > 1:
|
|
214
|
+
self.data["metadata"]["content_category"] = "site"
|
|
215
|
+
|
|
216
|
+
if environment:
|
|
217
|
+
pm_name = str(environment.package_manager)
|
|
218
|
+
pm_version_value = getattr(environment, pm_name, None)
|
|
219
|
+
if pm_version_value is None:
|
|
220
|
+
# Fallback: use pip version if available; otherwise empty string
|
|
221
|
+
pm_version_value = getattr(environment, "pip", "")
|
|
222
|
+
pm: ManifestDataPythonPackageManager = {
|
|
223
|
+
"name": pm_name,
|
|
224
|
+
"version": pm_version_value,
|
|
225
|
+
"package_file": environment.filename,
|
|
226
|
+
}
|
|
227
|
+
if getattr(environment, "package_manager_allow_uv", None) is not None:
|
|
228
|
+
pm["allow_uv"] = typing.cast(bool, environment.package_manager_allow_uv)
|
|
229
|
+
self.data["python"] = {"version": environment.python, "package_manager": pm}
|
|
230
|
+
|
|
231
|
+
if environment.python_version_requirement:
|
|
232
|
+
# If the environment has a python version requirement,
|
|
233
|
+
# add it to the manifest as environment.python.requires
|
|
234
|
+
manifest_environment = self.data.setdefault("environment", {})
|
|
235
|
+
manifest_environment["python"] = {"requires": environment.python_version_requirement}
|
|
236
|
+
|
|
237
|
+
if r_environment:
|
|
238
|
+
self.data["platform"] = r_environment.r_version
|
|
239
|
+
self.data["packages"] = cast("dict[str, ManifestDataRPackage]", r_environment.packages)
|
|
240
|
+
|
|
241
|
+
if image or env_management_py is not None or env_management_r is not None:
|
|
242
|
+
manifest_environment = self.data.setdefault("environment", {})
|
|
243
|
+
if image:
|
|
244
|
+
manifest_environment["image"] = image
|
|
245
|
+
if env_management_py is not None or env_management_r is not None:
|
|
246
|
+
manifest_environment["environment_management"] = {}
|
|
247
|
+
if env_management_py is not None:
|
|
248
|
+
manifest_environment["environment_management"]["python"] = env_management_py
|
|
249
|
+
if env_management_r is not None:
|
|
250
|
+
manifest_environment["environment_management"]["r"] = env_management_r
|
|
251
|
+
|
|
252
|
+
self.data["files"] = {}
|
|
253
|
+
if files:
|
|
254
|
+
self.data["files"] = files
|
|
255
|
+
|
|
256
|
+
@classmethod
|
|
257
|
+
def from_json(cls, json_str: str):
|
|
258
|
+
return cls(**json.loads(json_str))
|
|
259
|
+
|
|
260
|
+
@classmethod
|
|
261
|
+
def from_json_file(cls, json_path: str | Path):
|
|
262
|
+
with open(json_path) as json_file:
|
|
263
|
+
return cls(**json.load(json_file))
|
|
264
|
+
|
|
265
|
+
@property
|
|
266
|
+
def json(self):
|
|
267
|
+
return json.dumps(self.data, indent=2)
|
|
268
|
+
|
|
269
|
+
@property
|
|
270
|
+
def entrypoint(self):
|
|
271
|
+
if "metadata" not in self.data:
|
|
272
|
+
return None
|
|
273
|
+
if "entrypoint" in self.data["metadata"]:
|
|
274
|
+
return self.data["metadata"]["entrypoint"]
|
|
275
|
+
return None
|
|
276
|
+
|
|
277
|
+
@entrypoint.setter
|
|
278
|
+
def entrypoint(self, value: str):
|
|
279
|
+
self.data["metadata"]["entrypoint"] = value
|
|
280
|
+
|
|
281
|
+
@property
|
|
282
|
+
def primary_html(self):
|
|
283
|
+
if "metadata" not in self.data:
|
|
284
|
+
return None
|
|
285
|
+
if "primary_html" in self.data["metadata"]:
|
|
286
|
+
return self.data["metadata"]["primary_html"]
|
|
287
|
+
return None
|
|
288
|
+
|
|
289
|
+
@primary_html.setter
|
|
290
|
+
def primary_html(self, value: str):
|
|
291
|
+
self.data["metadata"]["primary_html"] = value
|
|
292
|
+
|
|
293
|
+
def add_file(self, path: str):
|
|
294
|
+
manifestPath = Path(path).as_posix()
|
|
295
|
+
self.data["files"][manifestPath] = {"checksum": file_checksum(path)}
|
|
296
|
+
return self
|
|
297
|
+
|
|
298
|
+
def discard_file(self, path: str):
|
|
299
|
+
if path in self.data["files"]:
|
|
300
|
+
del self.data["files"][path]
|
|
301
|
+
return self
|
|
302
|
+
|
|
303
|
+
def add_to_buffer(self, key: str, value: str):
|
|
304
|
+
self.buffer[key] = value
|
|
305
|
+
self.data["files"][key] = {"checksum": buffer_checksum(value)}
|
|
306
|
+
return self
|
|
307
|
+
|
|
308
|
+
def discard_from_buffer(self, key: str):
|
|
309
|
+
if key in self.buffer:
|
|
310
|
+
del self.buffer[key]
|
|
311
|
+
del self.data["files"][key]
|
|
312
|
+
return self
|
|
313
|
+
|
|
314
|
+
def require_entrypoint(self) -> str:
|
|
315
|
+
"""
|
|
316
|
+
If self.entrypoint is a string, return it; if it is None, raise an exception.
|
|
317
|
+
"""
|
|
318
|
+
if self.entrypoint is None:
|
|
319
|
+
raise RSConnectException("A valid entrypoint must be provided.")
|
|
320
|
+
return self.entrypoint
|
|
321
|
+
|
|
322
|
+
def get_manifest_files(self) -> dict[str, ManifestDataFile]:
|
|
323
|
+
new_data_files: dict[str, ManifestDataFile] = {}
|
|
324
|
+
deploy_dir: str
|
|
325
|
+
|
|
326
|
+
entrypoint = self.require_entrypoint()
|
|
327
|
+
if self.deploy_dir is not None:
|
|
328
|
+
deploy_dir = self.deploy_dir
|
|
329
|
+
elif entrypoint is not None and isfile(entrypoint):
|
|
330
|
+
deploy_dir = dirname(entrypoint)
|
|
331
|
+
else:
|
|
332
|
+
# TODO: This branch might be an error case. Need to investigate.
|
|
333
|
+
deploy_dir = entrypoint
|
|
334
|
+
|
|
335
|
+
for path in self.data["files"]:
|
|
336
|
+
rel_path = relpath(path, deploy_dir)
|
|
337
|
+
manifestPath = Path(rel_path).as_posix()
|
|
338
|
+
new_data_files[manifestPath] = self.data["files"][path]
|
|
339
|
+
return new_data_files
|
|
340
|
+
|
|
341
|
+
def get_manifest_files_from_buffer(self) -> dict[str, str]:
|
|
342
|
+
new_buffer: dict[str, str] = {}
|
|
343
|
+
deploy_dir: str
|
|
344
|
+
|
|
345
|
+
entrypoint = self.require_entrypoint()
|
|
346
|
+
if self.deploy_dir is not None:
|
|
347
|
+
deploy_dir = self.deploy_dir
|
|
348
|
+
elif entrypoint is not None and isfile(entrypoint):
|
|
349
|
+
deploy_dir = dirname(entrypoint)
|
|
350
|
+
else:
|
|
351
|
+
# TODO: This branch might be an error case. Need to investigate.
|
|
352
|
+
deploy_dir = entrypoint
|
|
353
|
+
|
|
354
|
+
for k, v in self.buffer.items():
|
|
355
|
+
rel_path = relpath(k, deploy_dir)
|
|
356
|
+
manifestPath = Path(rel_path).as_posix()
|
|
357
|
+
new_buffer[manifestPath] = v
|
|
358
|
+
return new_buffer
|
|
359
|
+
|
|
360
|
+
def get_relative_entrypoint(self) -> str:
|
|
361
|
+
entrypoint = self.require_entrypoint()
|
|
362
|
+
return basename(entrypoint)
|
|
363
|
+
|
|
364
|
+
def get_flattened_primary_html(self):
|
|
365
|
+
if self.primary_html is None:
|
|
366
|
+
raise RSConnectException("A valid primary_html must be provided.")
|
|
367
|
+
return relpath(self.primary_html, dirname(self.primary_html))
|
|
368
|
+
|
|
369
|
+
def get_flattened_copy(self):
|
|
370
|
+
new_manifest = deepcopy(self)
|
|
371
|
+
new_manifest.data["files"] = self.get_manifest_files()
|
|
372
|
+
new_manifest.buffer = self.get_manifest_files_from_buffer()
|
|
373
|
+
new_manifest.entrypoint = self.get_relative_entrypoint()
|
|
374
|
+
if self.primary_html:
|
|
375
|
+
new_manifest.primary_html = self.get_flattened_primary_html()
|
|
376
|
+
return new_manifest
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
class Bundle:
|
|
380
|
+
def __init__(self) -> None:
|
|
381
|
+
self.file_paths: set[str] = set()
|
|
382
|
+
self.buffer: dict[str, str] = {}
|
|
383
|
+
|
|
384
|
+
def add_file(self, filepath: str) -> None:
|
|
385
|
+
self.file_paths.add(filepath)
|
|
386
|
+
|
|
387
|
+
def discard_file(self, filepath: str) -> None:
|
|
388
|
+
self.file_paths.discard(filepath)
|
|
389
|
+
|
|
390
|
+
def to_file(self, deploy_dir: str) -> typing.IO[bytes]:
|
|
391
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
392
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
393
|
+
for fp in self.file_paths:
|
|
394
|
+
if Path(fp).name in self.buffer:
|
|
395
|
+
continue
|
|
396
|
+
rel_path = Path(fp).relative_to(deploy_dir)
|
|
397
|
+
logger.log(VERBOSE, "Adding file: %s", fp)
|
|
398
|
+
bundle.add(fp, arcname=rel_path)
|
|
399
|
+
for k, v in self.buffer.items():
|
|
400
|
+
buf = io.BytesIO(to_bytes(v))
|
|
401
|
+
file_info = tarfile.TarInfo(k)
|
|
402
|
+
file_info.size = len(buf.getvalue())
|
|
403
|
+
logger.log(VERBOSE, "Adding file: %s", k)
|
|
404
|
+
bundle.addfile(file_info, buf)
|
|
405
|
+
bundle_file.seek(0)
|
|
406
|
+
return bundle_file
|
|
407
|
+
|
|
408
|
+
def add_to_buffer(self, key: str, value: str):
|
|
409
|
+
self.buffer[key] = value
|
|
410
|
+
return self
|
|
411
|
+
|
|
412
|
+
def discard_from_buffer(self, key: str):
|
|
413
|
+
if key in self.buffer:
|
|
414
|
+
del self.buffer[key]
|
|
415
|
+
return self
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
# noinspection SpellCheckingInspection
|
|
419
|
+
def make_source_manifest(
|
|
420
|
+
app_mode: AppMode,
|
|
421
|
+
environment: Optional[Environment] = None,
|
|
422
|
+
entrypoint: Optional[str] = None,
|
|
423
|
+
quarto_inspection: Optional[QuartoInspectResult] = None,
|
|
424
|
+
image: Optional[str] = None,
|
|
425
|
+
env_management_py: Optional[bool] = None,
|
|
426
|
+
env_management_r: Optional[bool] = None,
|
|
427
|
+
r_environment: Optional[REnvironment] = None,
|
|
428
|
+
) -> ManifestData:
|
|
429
|
+
manifest: Manifest = Manifest(
|
|
430
|
+
app_mode=app_mode,
|
|
431
|
+
environment=environment,
|
|
432
|
+
r_environment=r_environment,
|
|
433
|
+
entrypoint=entrypoint,
|
|
434
|
+
quarto_inspection=quarto_inspection,
|
|
435
|
+
image=image,
|
|
436
|
+
env_management_py=env_management_py,
|
|
437
|
+
env_management_r=env_management_r,
|
|
438
|
+
)
|
|
439
|
+
return manifest.data
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def manifest_add_file(manifest: ManifestData, rel_path: str, base_dir: str) -> None:
|
|
443
|
+
"""Add the specified file to the manifest files section
|
|
444
|
+
|
|
445
|
+
The file must be specified as a pathname relative to the notebook directory.
|
|
446
|
+
"""
|
|
447
|
+
path = join(base_dir, rel_path) if os.path.isdir(base_dir) else rel_path
|
|
448
|
+
if "files" not in manifest:
|
|
449
|
+
manifest["files"] = {}
|
|
450
|
+
manifestPath = Path(rel_path).as_posix()
|
|
451
|
+
manifest["files"][manifestPath] = {"checksum": file_checksum(path)}
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def manifest_add_buffer(manifest: ManifestData, filename: str, buf: str | bytes) -> None:
|
|
455
|
+
"""Add the specified in-memory buffer to the manifest files section"""
|
|
456
|
+
manifest["files"][filename] = {"checksum": buffer_checksum(buf)}
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def make_hasher():
|
|
460
|
+
try:
|
|
461
|
+
return hashlib.md5()
|
|
462
|
+
except Exception:
|
|
463
|
+
# md5 is not available in FIPS mode, see if the usedforsecurity option is available
|
|
464
|
+
# (it was added in python 3.9). We set usedforsecurity=False since we are only
|
|
465
|
+
# using this for a file upload integrity check.
|
|
466
|
+
return hashlib.md5(usedforsecurity=False)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def file_checksum(path: str | Path) -> str:
|
|
470
|
+
"""Calculate the md5 hex digest of the specified file"""
|
|
471
|
+
with open(path, "rb") as f:
|
|
472
|
+
m = make_hasher()
|
|
473
|
+
chunk_size = 64 * 1024
|
|
474
|
+
|
|
475
|
+
chunk = f.read(chunk_size)
|
|
476
|
+
while chunk:
|
|
477
|
+
m.update(chunk)
|
|
478
|
+
chunk = f.read(chunk_size)
|
|
479
|
+
return m.hexdigest()
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def buffer_checksum(buf: str | bytes) -> str:
|
|
483
|
+
"""Calculate the md5 hex digest of a buffer (str or bytes)"""
|
|
484
|
+
m = make_hasher()
|
|
485
|
+
m.update(to_bytes(buf))
|
|
486
|
+
return m.hexdigest()
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def to_bytes(s: str | bytes) -> bytes:
|
|
490
|
+
if isinstance(s, bytes):
|
|
491
|
+
return s
|
|
492
|
+
elif isinstance(s, str):
|
|
493
|
+
return s.encode("utf-8")
|
|
494
|
+
logger.warning("can't encode to bytes: %s" % type(s).__name__)
|
|
495
|
+
return s
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def bundle_add_file(bundle: tarfile.TarFile, rel_path: str, base_dir: str) -> None:
|
|
499
|
+
"""Add the specified file to the tarball.
|
|
500
|
+
|
|
501
|
+
The file path is relative to the notebook directory.
|
|
502
|
+
"""
|
|
503
|
+
path = join(base_dir, rel_path) if os.path.isdir(base_dir) else rel_path
|
|
504
|
+
logger.log(VERBOSE, "Adding file: %s", path)
|
|
505
|
+
bundle.add(path, arcname=rel_path)
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def bundle_add_buffer(bundle: tarfile.TarFile, filename: str, contents: str | bytes) -> None:
|
|
509
|
+
"""Add an in-memory buffer to the tarball.
|
|
510
|
+
|
|
511
|
+
`contents` may be a string or bytes object
|
|
512
|
+
"""
|
|
513
|
+
logger.log(VERBOSE, "Adding file: %s", filename)
|
|
514
|
+
buf = io.BytesIO(to_bytes(contents))
|
|
515
|
+
file_info = tarfile.TarInfo(filename)
|
|
516
|
+
file_info.size = len(buf.getvalue())
|
|
517
|
+
bundle.addfile(file_info, buf)
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def write_manifest(
|
|
521
|
+
relative_dir: str,
|
|
522
|
+
nb_name: str,
|
|
523
|
+
environment: Environment,
|
|
524
|
+
output_dir: str,
|
|
525
|
+
hide_all_input: bool = False,
|
|
526
|
+
hide_tagged_input: bool = False,
|
|
527
|
+
image: Optional[str] = None,
|
|
528
|
+
env_management_py: Optional[bool] = None,
|
|
529
|
+
env_management_r: Optional[bool] = None,
|
|
530
|
+
) -> tuple[list[str], list[str]]:
|
|
531
|
+
"""Create a manifest for source publishing the specified notebook.
|
|
532
|
+
|
|
533
|
+
The manifest will be written to `manifest.json` in the output directory..
|
|
534
|
+
A requirements.txt file will be created if one does not exist.
|
|
535
|
+
|
|
536
|
+
Returns the list of filenames written.
|
|
537
|
+
"""
|
|
538
|
+
manifest_filename = "manifest.json"
|
|
539
|
+
manifest = make_source_manifest(
|
|
540
|
+
AppModes.JUPYTER_NOTEBOOK, environment, nb_name, None, image, env_management_py, env_management_r
|
|
541
|
+
)
|
|
542
|
+
if hide_all_input:
|
|
543
|
+
if "jupyter" not in manifest:
|
|
544
|
+
manifest["jupyter"] = {}
|
|
545
|
+
manifest["jupyter"].update({"hide_all_input": hide_all_input})
|
|
546
|
+
if hide_tagged_input:
|
|
547
|
+
if "jupyter" not in manifest:
|
|
548
|
+
manifest["jupyter"] = {}
|
|
549
|
+
manifest["jupyter"].update({"hide_tagged_input": hide_tagged_input})
|
|
550
|
+
manifest_file = join(output_dir, manifest_filename)
|
|
551
|
+
created: list[str] = []
|
|
552
|
+
skipped: list[str] = []
|
|
553
|
+
|
|
554
|
+
manifest_relative_path = join(relative_dir, manifest_filename)
|
|
555
|
+
if exists(manifest_file):
|
|
556
|
+
skipped.append(manifest_relative_path)
|
|
557
|
+
else:
|
|
558
|
+
with open(manifest_file, "w") as f:
|
|
559
|
+
f.write(json.dumps(manifest, indent=2))
|
|
560
|
+
created.append(manifest_relative_path)
|
|
561
|
+
logger.debug("wrote manifest file: %s", manifest_file)
|
|
562
|
+
|
|
563
|
+
environment_filename = environment.filename
|
|
564
|
+
environment_file = join(output_dir, environment_filename)
|
|
565
|
+
environment_relative_path = join(relative_dir, environment_filename)
|
|
566
|
+
if environment.source == "file":
|
|
567
|
+
skipped.append(environment_relative_path)
|
|
568
|
+
else:
|
|
569
|
+
with open(environment_file, "w") as f:
|
|
570
|
+
f.write(environment.contents)
|
|
571
|
+
created.append(environment_relative_path)
|
|
572
|
+
logger.debug("wrote environment file: %s", environment_file)
|
|
573
|
+
|
|
574
|
+
return created, skipped
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def list_files(
|
|
578
|
+
base_dir: str,
|
|
579
|
+
include_sub_dirs: bool,
|
|
580
|
+
walk: Callable[[str], Iterator[tuple[str, list[str], list[str]]]] = os.walk,
|
|
581
|
+
) -> list[str]:
|
|
582
|
+
"""List the files in the directory at path.
|
|
583
|
+
|
|
584
|
+
If include_sub_dirs is True, recursively list
|
|
585
|
+
files in subdirectories.
|
|
586
|
+
|
|
587
|
+
Returns an iterable of file paths relative to base_dir.
|
|
588
|
+
"""
|
|
589
|
+
skip_dirs = [".ipynb_checkpoints", ".git"]
|
|
590
|
+
|
|
591
|
+
def iter_files():
|
|
592
|
+
for root, sub_dirs, files in walk(base_dir):
|
|
593
|
+
if include_sub_dirs:
|
|
594
|
+
for skip in skip_dirs:
|
|
595
|
+
if skip in sub_dirs:
|
|
596
|
+
sub_dirs.remove(skip)
|
|
597
|
+
else:
|
|
598
|
+
# tell walk not to traverse any subdirectories
|
|
599
|
+
sub_dirs[:] = []
|
|
600
|
+
|
|
601
|
+
for filename in files:
|
|
602
|
+
yield relpath(join(root, filename), base_dir)
|
|
603
|
+
|
|
604
|
+
return list(iter_files())
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def make_notebook_source_bundle(
|
|
608
|
+
file: str,
|
|
609
|
+
environment: Environment,
|
|
610
|
+
extra_files: Sequence[str],
|
|
611
|
+
hide_all_input: bool,
|
|
612
|
+
hide_tagged_input: bool,
|
|
613
|
+
image: Optional[str] = None,
|
|
614
|
+
env_management_py: Optional[bool] = None,
|
|
615
|
+
env_management_r: Optional[bool] = None,
|
|
616
|
+
r_environment: Optional[REnvironment] = None,
|
|
617
|
+
) -> IO[bytes]:
|
|
618
|
+
"""Create a bundle containing the specified notebook and python environment.
|
|
619
|
+
|
|
620
|
+
Returns a file-like object containing the bundle tarball.
|
|
621
|
+
|
|
622
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
623
|
+
"""
|
|
624
|
+
if extra_files is None:
|
|
625
|
+
extra_files = []
|
|
626
|
+
base_dir = dirname(file)
|
|
627
|
+
nb_name = basename(file)
|
|
628
|
+
|
|
629
|
+
manifest = make_source_manifest(
|
|
630
|
+
AppModes.JUPYTER_NOTEBOOK,
|
|
631
|
+
environment,
|
|
632
|
+
nb_name,
|
|
633
|
+
None,
|
|
634
|
+
image,
|
|
635
|
+
env_management_py,
|
|
636
|
+
env_management_r,
|
|
637
|
+
r_environment,
|
|
638
|
+
)
|
|
639
|
+
if hide_all_input:
|
|
640
|
+
if "jupyter" not in manifest:
|
|
641
|
+
manifest["jupyter"] = {}
|
|
642
|
+
manifest["jupyter"].update({"hide_all_input": hide_all_input})
|
|
643
|
+
if hide_tagged_input:
|
|
644
|
+
if "jupyter" not in manifest:
|
|
645
|
+
manifest["jupyter"] = {}
|
|
646
|
+
manifest["jupyter"].update({"hide_tagged_input": hide_tagged_input})
|
|
647
|
+
manifest_add_file(manifest, nb_name, base_dir)
|
|
648
|
+
manifest_add_buffer(manifest, environment.filename, environment.contents)
|
|
649
|
+
|
|
650
|
+
if extra_files:
|
|
651
|
+
skip = [nb_name, environment.filename, "manifest.json"]
|
|
652
|
+
extra_files = sorted(list(set(extra_files) - set(skip)))
|
|
653
|
+
|
|
654
|
+
for rel_path in extra_files:
|
|
655
|
+
manifest_add_file(manifest, rel_path, base_dir)
|
|
656
|
+
|
|
657
|
+
logger.debug("manifest: %r", manifest)
|
|
658
|
+
|
|
659
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
660
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
661
|
+
# add the manifest first in case we want to partially untar the bundle for inspection
|
|
662
|
+
bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2))
|
|
663
|
+
bundle_add_buffer(bundle, environment.filename, environment.contents)
|
|
664
|
+
bundle_add_file(bundle, nb_name, base_dir)
|
|
665
|
+
|
|
666
|
+
for rel_path in extra_files:
|
|
667
|
+
bundle_add_file(bundle, rel_path, base_dir)
|
|
668
|
+
|
|
669
|
+
bundle_file.seek(0)
|
|
670
|
+
return bundle_file
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def make_quarto_source_bundle(
|
|
674
|
+
file_or_directory: str,
|
|
675
|
+
inspect: QuartoInspectResult,
|
|
676
|
+
app_mode: AppMode,
|
|
677
|
+
environment: Optional[Environment],
|
|
678
|
+
extra_files: Sequence[str],
|
|
679
|
+
excludes: Sequence[str],
|
|
680
|
+
image: Optional[str] = None,
|
|
681
|
+
env_management_py: Optional[bool] = None,
|
|
682
|
+
env_management_r: Optional[bool] = None,
|
|
683
|
+
r_environment: Optional[REnvironment] = None,
|
|
684
|
+
) -> typing.IO[bytes]:
|
|
685
|
+
"""
|
|
686
|
+
Create a bundle containing the specified Quarto content and (optional)
|
|
687
|
+
python environment.
|
|
688
|
+
|
|
689
|
+
Returns a file-like object containing the bundle tarball.
|
|
690
|
+
|
|
691
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
692
|
+
"""
|
|
693
|
+
manifest, relevant_files = make_quarto_manifest(
|
|
694
|
+
file_or_directory,
|
|
695
|
+
inspect,
|
|
696
|
+
app_mode,
|
|
697
|
+
environment,
|
|
698
|
+
extra_files,
|
|
699
|
+
excludes,
|
|
700
|
+
image,
|
|
701
|
+
env_management_py,
|
|
702
|
+
env_management_r,
|
|
703
|
+
r_environment,
|
|
704
|
+
)
|
|
705
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
706
|
+
|
|
707
|
+
base_dir = file_or_directory
|
|
708
|
+
if not isdir(file_or_directory):
|
|
709
|
+
base_dir = dirname(file_or_directory)
|
|
710
|
+
|
|
711
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
712
|
+
bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2))
|
|
713
|
+
if environment:
|
|
714
|
+
bundle_add_buffer(bundle, environment.filename, environment.contents)
|
|
715
|
+
|
|
716
|
+
for rel_path in relevant_files:
|
|
717
|
+
bundle_add_file(bundle, rel_path, base_dir)
|
|
718
|
+
|
|
719
|
+
# rewind file pointer
|
|
720
|
+
bundle_file.seek(0)
|
|
721
|
+
|
|
722
|
+
return bundle_file
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def make_html_manifest(
|
|
726
|
+
filename: str,
|
|
727
|
+
) -> ManifestData:
|
|
728
|
+
# noinspection SpellCheckingInspection
|
|
729
|
+
manifest = Manifest(
|
|
730
|
+
metadata=ManifestDataMetadata(
|
|
731
|
+
appmode="static",
|
|
732
|
+
primary_html=filename,
|
|
733
|
+
)
|
|
734
|
+
)
|
|
735
|
+
return manifest.data
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def make_notebook_html_bundle(
|
|
739
|
+
filename: str,
|
|
740
|
+
python: str,
|
|
741
|
+
hide_all_input: bool,
|
|
742
|
+
hide_tagged_input: bool,
|
|
743
|
+
check_output: Callable[..., bytes] = subprocess.check_output,
|
|
744
|
+
) -> typing.IO[bytes]:
|
|
745
|
+
# noinspection SpellCheckingInspection
|
|
746
|
+
cmd = [
|
|
747
|
+
python,
|
|
748
|
+
"-m",
|
|
749
|
+
"nbconvert",
|
|
750
|
+
"--execute",
|
|
751
|
+
"--stdout",
|
|
752
|
+
"--log-level=ERROR",
|
|
753
|
+
"--to=html",
|
|
754
|
+
filename,
|
|
755
|
+
]
|
|
756
|
+
if hide_all_input and hide_tagged_input or hide_all_input:
|
|
757
|
+
cmd.append("--no-input")
|
|
758
|
+
elif hide_tagged_input:
|
|
759
|
+
version = check_output([python, "--version"]).decode("utf-8")
|
|
760
|
+
if version >= "Python 3":
|
|
761
|
+
cmd.append("--TagRemovePreprocessor.remove_input_tags=hide_input")
|
|
762
|
+
else:
|
|
763
|
+
cmd.append("--TagRemovePreprocessor.remove_input_tags=['hide_input']")
|
|
764
|
+
try:
|
|
765
|
+
output = check_output(cmd)
|
|
766
|
+
except subprocess.CalledProcessError:
|
|
767
|
+
raise
|
|
768
|
+
|
|
769
|
+
nb_name = basename(filename)
|
|
770
|
+
filename = splitext(nb_name)[0] + ".html"
|
|
771
|
+
|
|
772
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
773
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
774
|
+
bundle_add_buffer(bundle, filename, output)
|
|
775
|
+
|
|
776
|
+
# manifest
|
|
777
|
+
manifest = make_html_manifest(filename)
|
|
778
|
+
bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2))
|
|
779
|
+
|
|
780
|
+
# rewind file pointer
|
|
781
|
+
bundle_file.seek(0)
|
|
782
|
+
return bundle_file
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def keep_manifest_specified_file(relative_path: str, ignore_path_set: set[Path] = directories_to_ignore) -> bool:
|
|
786
|
+
"""
|
|
787
|
+
A helper to see if the relative path given, which is assumed to have come
|
|
788
|
+
from a manifest.json file, should be kept or ignored.
|
|
789
|
+
|
|
790
|
+
:param relative_path: the relative path name to check.
|
|
791
|
+
:return: True, if the path should kept or False, if it should be ignored.
|
|
792
|
+
"""
|
|
793
|
+
p = Path(relative_path)
|
|
794
|
+
for parent in p.parents:
|
|
795
|
+
if parent in ignore_path_set:
|
|
796
|
+
return False
|
|
797
|
+
if p in ignore_path_set:
|
|
798
|
+
return False
|
|
799
|
+
return True
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _default_title_from_manifest(the_manifest: ManifestData, manifest_file: str | Path) -> str:
|
|
803
|
+
"""
|
|
804
|
+
Produce a default content title from the contents of a manifest.
|
|
805
|
+
"""
|
|
806
|
+
filename = None
|
|
807
|
+
|
|
808
|
+
metadata = the_manifest.get("metadata")
|
|
809
|
+
if metadata:
|
|
810
|
+
# noinspection SpellCheckingInspection
|
|
811
|
+
filename = metadata.get("entrypoint") or metadata.get("primary_rmd") or metadata.get("primary_html")
|
|
812
|
+
# If the manifest is for an API, revert to using the parent directory.
|
|
813
|
+
if filename and _module_pattern.match(filename):
|
|
814
|
+
filename = None
|
|
815
|
+
return _default_title(filename or dirname(manifest_file))
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def read_manifest_app_mode(file: str | Path) -> AppMode:
|
|
819
|
+
source_manifest, _ = read_manifest_file(file)
|
|
820
|
+
# noinspection SpellCheckingInspection
|
|
821
|
+
app_mode = AppModes.get_by_name(source_manifest["metadata"]["appmode"])
|
|
822
|
+
return app_mode
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
def default_title_from_manifest(file: str | Path) -> str:
|
|
826
|
+
source_manifest, _ = read_manifest_file(file)
|
|
827
|
+
title = _default_title_from_manifest(source_manifest, file)
|
|
828
|
+
return title
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def read_manifest_file(manifest_path: str | Path) -> tuple[ManifestData, str]:
|
|
832
|
+
"""
|
|
833
|
+
Read a manifest's content from its file. The content is provided as both a
|
|
834
|
+
raw string and a parsed dictionary.
|
|
835
|
+
|
|
836
|
+
:param manifest_path: the path to the file to read.
|
|
837
|
+
:return: the parsed manifest data and the raw file content as a string.
|
|
838
|
+
"""
|
|
839
|
+
with open(manifest_path, "rb") as f:
|
|
840
|
+
raw_manifest = f.read().decode("utf-8")
|
|
841
|
+
manifest = json.loads(raw_manifest)
|
|
842
|
+
|
|
843
|
+
return manifest, raw_manifest
|
|
844
|
+
|
|
845
|
+
|
|
846
|
+
def _find_manifest_member(tar: tarfile.TarFile) -> Optional[tarfile.TarInfo]:
|
|
847
|
+
"""
|
|
848
|
+
Locate the manifest.json member within a bundle tarball, mirroring the way
|
|
849
|
+
Connect collapses single-subdirectory bundles when it extracts them.
|
|
850
|
+
|
|
851
|
+
Connect repeatedly descends while the extraction root contains exactly one
|
|
852
|
+
entry and that entry is a directory, moving its contents up a level. So a
|
|
853
|
+
downloaded bundle may legitimately store manifest.json under a nested
|
|
854
|
+
directory (e.g. "bundle/manifest.json") rather than at the top level.
|
|
855
|
+
|
|
856
|
+
:return: the manifest.json member, or None if no manifest could be located.
|
|
857
|
+
"""
|
|
858
|
+
file_names = {(m.name[2:] if m.name.startswith("./") else m.name): m for m in tar.getmembers() if m.isfile()}
|
|
859
|
+
|
|
860
|
+
prefix = ""
|
|
861
|
+
while True:
|
|
862
|
+
member = file_names.get(prefix + "manifest.json")
|
|
863
|
+
if member is not None:
|
|
864
|
+
return member
|
|
865
|
+
|
|
866
|
+
# Look at the entries directly under the current prefix.
|
|
867
|
+
remaining = [name[len(prefix) :] for name in file_names if name.startswith(prefix)]
|
|
868
|
+
entries = {name.split("/", 1)[0] for name in remaining}
|
|
869
|
+
|
|
870
|
+
# Only descend when this level holds exactly one entry and it is a
|
|
871
|
+
# directory (i.e. no file is named exactly that entry).
|
|
872
|
+
if len(entries) != 1:
|
|
873
|
+
return None
|
|
874
|
+
entry = next(iter(entries))
|
|
875
|
+
if entry in remaining:
|
|
876
|
+
return None
|
|
877
|
+
prefix = prefix + entry + "/"
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
def read_bundle_manifest(bundle_path: str | Path) -> ManifestData:
|
|
881
|
+
"""
|
|
882
|
+
Read and parse the manifest.json contained in a bundle tarball without
|
|
883
|
+
extracting the whole bundle.
|
|
884
|
+
|
|
885
|
+
:param bundle_path: the path to a bundle .tar.gz file.
|
|
886
|
+
:return: the parsed manifest data.
|
|
887
|
+
"""
|
|
888
|
+
with tarfile.open(name=str(bundle_path), mode="r:gz") as tar:
|
|
889
|
+
member = _find_manifest_member(tar)
|
|
890
|
+
extracted = tar.extractfile(member) if member is not None else None
|
|
891
|
+
if extracted is None:
|
|
892
|
+
raise RSConnectException('Bundle "%s" does not contain a manifest.json file.' % bundle_path)
|
|
893
|
+
raw_manifest = extracted.read().decode("utf-8")
|
|
894
|
+
|
|
895
|
+
return json.loads(raw_manifest)
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def read_bundle_app_mode(bundle_path: str | Path) -> AppMode:
|
|
899
|
+
source_manifest = read_bundle_manifest(bundle_path)
|
|
900
|
+
# noinspection SpellCheckingInspection
|
|
901
|
+
return AppModes.get_by_name(source_manifest["metadata"]["appmode"])
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
def default_title_from_bundle(bundle_path: str | Path) -> str:
|
|
905
|
+
source_manifest = read_bundle_manifest(bundle_path)
|
|
906
|
+
|
|
907
|
+
# Prefer the manifest's entry point / primary file, mirroring how a manifest
|
|
908
|
+
# deployment derives its title.
|
|
909
|
+
filename = None
|
|
910
|
+
metadata = source_manifest.get("metadata")
|
|
911
|
+
if metadata:
|
|
912
|
+
# noinspection SpellCheckingInspection
|
|
913
|
+
filename = metadata.get("entrypoint") or metadata.get("primary_rmd") or metadata.get("primary_html")
|
|
914
|
+
# If the manifest is for a module-style API entry point, there is no
|
|
915
|
+
# useful filename to derive a title from.
|
|
916
|
+
if filename and _module_pattern.match(filename):
|
|
917
|
+
filename = None
|
|
918
|
+
|
|
919
|
+
# When the manifest has no usable filename, fall back to the bundle's own
|
|
920
|
+
# file name (e.g. "mycontent" from "mycontent.tar.gz") rather than the
|
|
921
|
+
# directory the bundle happens to live in, which is unrelated to the content.
|
|
922
|
+
# Connect always produces .tar.gz bundles, and the name may legitimately
|
|
923
|
+
# contain dots (e.g. "my.cool.api"), so only the .tar.gz extension is stripped
|
|
924
|
+
# rather than splitting off a second "extension".
|
|
925
|
+
if not filename:
|
|
926
|
+
name = basename(str(bundle_path))
|
|
927
|
+
if name.lower().endswith(".tar.gz"):
|
|
928
|
+
name = name[: -len(".tar.gz")]
|
|
929
|
+
return _enforce_title_length(name)
|
|
930
|
+
|
|
931
|
+
return _default_title(filename)
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
def open_bundle(bundle_path: str | Path) -> typing.IO[bytes]:
|
|
935
|
+
"""Open an existing bundle tarball so it can be uploaded as-is.
|
|
936
|
+
|
|
937
|
+
This exists to plug into ``RSConnectExecutor.make_bundle``, which expects a
|
|
938
|
+
callable that returns the bundle as a file-like object (e.g.
|
|
939
|
+
``make_manifest_bundle``, which builds a tarball). For ``deploy bundle`` we
|
|
940
|
+
already have a finished ``.tar.gz`` on disk, so the "builder" is just an
|
|
941
|
+
open() — no tarball is constructed. Routing through ``make_bundle`` keeps the
|
|
942
|
+
deployment-name setup and upload flow identical to the other deploy commands.
|
|
943
|
+
"""
|
|
944
|
+
return open(bundle_path, "rb")
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
def make_manifest_bundle(manifest_path: str | Path) -> typing.IO[bytes]:
|
|
948
|
+
"""Create a bundle, given a manifest.
|
|
949
|
+
|
|
950
|
+
:return: a file-like object containing the bundle tarball.
|
|
951
|
+
"""
|
|
952
|
+
manifest, raw_manifest = read_manifest_file(manifest_path)
|
|
953
|
+
|
|
954
|
+
base_dir = dirname(manifest_path)
|
|
955
|
+
files = list(filter(keep_manifest_specified_file, manifest.get("files", {}).keys()))
|
|
956
|
+
|
|
957
|
+
if "manifest.json" in files:
|
|
958
|
+
# this will be created
|
|
959
|
+
files.remove("manifest.json")
|
|
960
|
+
|
|
961
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
962
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
963
|
+
# add the manifest first in case we want to partially untar the bundle for inspection
|
|
964
|
+
bundle_add_buffer(bundle, "manifest.json", raw_manifest)
|
|
965
|
+
|
|
966
|
+
for rel_path in files:
|
|
967
|
+
bundle_add_file(bundle, rel_path, base_dir)
|
|
968
|
+
|
|
969
|
+
# rewind file pointer
|
|
970
|
+
bundle_file.seek(0)
|
|
971
|
+
|
|
972
|
+
return bundle_file
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def create_glob_set(directory: str | Path, excludes: Sequence[str]) -> GlobSet:
|
|
976
|
+
"""
|
|
977
|
+
Takes a list of glob strings and produces a GlobSet for path matching.
|
|
978
|
+
|
|
979
|
+
**Note:** we don't use Python's glob support because it takes way too
|
|
980
|
+
long to run when large file trees are involved in conjunction with the
|
|
981
|
+
'**' pattern.
|
|
982
|
+
|
|
983
|
+
:param directory: the directory the globs are relative to.
|
|
984
|
+
:param excludes: the list of globs to expand.
|
|
985
|
+
:return: a GlobSet ready for path matching.
|
|
986
|
+
"""
|
|
987
|
+
work: list[str] = []
|
|
988
|
+
if excludes:
|
|
989
|
+
for pattern in excludes:
|
|
990
|
+
file_pattern = join(directory, pattern)
|
|
991
|
+
# Special handling, if they gave us just a dir then "do the right thing".
|
|
992
|
+
if isdir(file_pattern):
|
|
993
|
+
file_pattern = join(file_pattern, "**/*")
|
|
994
|
+
work.append(file_pattern)
|
|
995
|
+
|
|
996
|
+
return GlobSet(work)
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def make_api_manifest(
|
|
1000
|
+
directory: str,
|
|
1001
|
+
entry_point: str,
|
|
1002
|
+
app_mode: AppMode,
|
|
1003
|
+
environment: Environment,
|
|
1004
|
+
extra_files: Sequence[str],
|
|
1005
|
+
excludes: Sequence[str],
|
|
1006
|
+
image: Optional[str] = None,
|
|
1007
|
+
env_management_py: Optional[bool] = None,
|
|
1008
|
+
env_management_r: Optional[bool] = None,
|
|
1009
|
+
r_environment: Optional[REnvironment] = None,
|
|
1010
|
+
) -> tuple[ManifestData, list[str]]:
|
|
1011
|
+
"""
|
|
1012
|
+
Makes a manifest for an API.
|
|
1013
|
+
|
|
1014
|
+
:param directory: the directory containing the files to deploy.
|
|
1015
|
+
:param entry_point: the main entry point for the API.
|
|
1016
|
+
:param app_mode: the app mode to use.
|
|
1017
|
+
:param environment: the Python environment information.
|
|
1018
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1019
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1020
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
1021
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
1022
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1023
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
1024
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1025
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
1026
|
+
:return: the manifest and a list of the files involved.
|
|
1027
|
+
"""
|
|
1028
|
+
if is_environment_dir(directory):
|
|
1029
|
+
excludes = list(excludes or []) + ["bin/", "lib/", "Lib/", "Scripts/", "Include/"]
|
|
1030
|
+
|
|
1031
|
+
extra_files = extra_files or []
|
|
1032
|
+
skip = [environment.filename, "manifest.json"]
|
|
1033
|
+
extra_files = sorted(list(set(extra_files) - set(skip)))
|
|
1034
|
+
|
|
1035
|
+
# Don't include these top-level files.
|
|
1036
|
+
excludes = list(excludes) if excludes else []
|
|
1037
|
+
excludes.append("manifest.json")
|
|
1038
|
+
excludes.append(environment.filename)
|
|
1039
|
+
excludes.extend(list_environment_dirs(directory))
|
|
1040
|
+
|
|
1041
|
+
relevant_files = create_file_list(directory, extra_files, excludes)
|
|
1042
|
+
manifest = make_source_manifest(
|
|
1043
|
+
app_mode,
|
|
1044
|
+
environment,
|
|
1045
|
+
entry_point,
|
|
1046
|
+
None,
|
|
1047
|
+
image,
|
|
1048
|
+
env_management_py,
|
|
1049
|
+
env_management_r,
|
|
1050
|
+
r_environment,
|
|
1051
|
+
)
|
|
1052
|
+
|
|
1053
|
+
manifest_add_buffer(manifest, environment.filename, environment.contents)
|
|
1054
|
+
|
|
1055
|
+
for rel_path in relevant_files:
|
|
1056
|
+
manifest_add_file(manifest, rel_path, directory)
|
|
1057
|
+
|
|
1058
|
+
return manifest, relevant_files
|
|
1059
|
+
|
|
1060
|
+
|
|
1061
|
+
def create_html_manifest(
|
|
1062
|
+
path: str,
|
|
1063
|
+
entrypoint: Optional[str],
|
|
1064
|
+
extra_files: Sequence[str],
|
|
1065
|
+
excludes: Sequence[str],
|
|
1066
|
+
) -> Manifest:
|
|
1067
|
+
"""
|
|
1068
|
+
Creates and writes a manifest.json file for the given path.
|
|
1069
|
+
|
|
1070
|
+
:param path: the file, or the directory containing the files to deploy.
|
|
1071
|
+
:param entrypoint: the main entry point for the API.
|
|
1072
|
+
:param environment: the Python environment to start with. This should be what's
|
|
1073
|
+
returned by the inspect_environment() function.
|
|
1074
|
+
:param app_mode: the application mode to assume. If this is None, the extension
|
|
1075
|
+
portion of the entry point file name will be used to derive one. Previous default = None.
|
|
1076
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
1077
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1078
|
+
:return: the manifest data structure.
|
|
1079
|
+
"""
|
|
1080
|
+
if not path:
|
|
1081
|
+
raise RSConnectException("A valid path must be provided.")
|
|
1082
|
+
extra_files = list(extra_files) if extra_files else []
|
|
1083
|
+
entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/html")
|
|
1084
|
+
|
|
1085
|
+
deploy_dir = guess_deploy_dir(path, entrypoint)
|
|
1086
|
+
if len(entrypoint_candidates) <= 0:
|
|
1087
|
+
if entrypoint is None:
|
|
1088
|
+
raise RSConnectException("No valid entrypoint found.")
|
|
1089
|
+
entrypoint = abs_entrypoint(path, entrypoint)
|
|
1090
|
+
elif len(entrypoint_candidates) == 1:
|
|
1091
|
+
if entrypoint:
|
|
1092
|
+
entrypoint = abs_entrypoint(path, entrypoint)
|
|
1093
|
+
else:
|
|
1094
|
+
entrypoint = entrypoint_candidates[0]
|
|
1095
|
+
else: # len(entrypoint_candidates) > 1:
|
|
1096
|
+
if entrypoint is None:
|
|
1097
|
+
raise RSConnectException("No valid entrypoint found.")
|
|
1098
|
+
entrypoint = abs_entrypoint(path, entrypoint)
|
|
1099
|
+
|
|
1100
|
+
extra_files = validate_extra_files(deploy_dir, extra_files, use_abspath=True)
|
|
1101
|
+
excludes = list(excludes) if excludes else []
|
|
1102
|
+
excludes.extend(["manifest.json"])
|
|
1103
|
+
excludes.extend(list_environment_dirs(deploy_dir))
|
|
1104
|
+
|
|
1105
|
+
manifest = Manifest(
|
|
1106
|
+
app_mode=AppModes.STATIC,
|
|
1107
|
+
entrypoint=entrypoint,
|
|
1108
|
+
primary_html=entrypoint,
|
|
1109
|
+
)
|
|
1110
|
+
manifest.deploy_dir = deploy_dir
|
|
1111
|
+
|
|
1112
|
+
file_list = create_file_list(path, extra_files, excludes, use_abspath=True)
|
|
1113
|
+
for abs_path in file_list:
|
|
1114
|
+
manifest.add_file(abs_path)
|
|
1115
|
+
|
|
1116
|
+
return manifest
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def make_tensorflow_manifest(
|
|
1120
|
+
directory: str,
|
|
1121
|
+
extra_files: Sequence[str],
|
|
1122
|
+
excludes: Sequence[str],
|
|
1123
|
+
image: Optional[str] = None,
|
|
1124
|
+
) -> ManifestData:
|
|
1125
|
+
"""
|
|
1126
|
+
Creates and writes a manifest.json file for the given path.
|
|
1127
|
+
|
|
1128
|
+
:param directory the directory containing the TensorFlow model.
|
|
1129
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
1130
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1131
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
1132
|
+
:return: the manifest data structure.
|
|
1133
|
+
"""
|
|
1134
|
+
if not directory:
|
|
1135
|
+
raise RSConnectException("A valid directory must be provided.")
|
|
1136
|
+
extra_files = list(extra_files) if extra_files else []
|
|
1137
|
+
|
|
1138
|
+
extra_files = validate_extra_files(directory, extra_files, use_abspath=True)
|
|
1139
|
+
excludes = list(excludes) if excludes else []
|
|
1140
|
+
excludes.extend(["manifest.json"])
|
|
1141
|
+
excludes.extend(list_environment_dirs(directory))
|
|
1142
|
+
|
|
1143
|
+
manifest = make_source_manifest(
|
|
1144
|
+
app_mode=AppModes.TENSORFLOW,
|
|
1145
|
+
image=image,
|
|
1146
|
+
)
|
|
1147
|
+
|
|
1148
|
+
file_list = create_file_list(directory, extra_files, excludes)
|
|
1149
|
+
for rel_path in file_list:
|
|
1150
|
+
manifest_add_file(manifest, rel_path, directory)
|
|
1151
|
+
return manifest
|
|
1152
|
+
|
|
1153
|
+
|
|
1154
|
+
def make_html_bundle(
|
|
1155
|
+
path: str,
|
|
1156
|
+
entrypoint: Optional[str],
|
|
1157
|
+
extra_files: Sequence[str],
|
|
1158
|
+
excludes: Sequence[str],
|
|
1159
|
+
) -> typing.IO[bytes]:
|
|
1160
|
+
"""
|
|
1161
|
+
Create an html bundle, given a path and/or entrypoint.
|
|
1162
|
+
|
|
1163
|
+
The bundle contains a manifest.json file created for the given notebook entrypoint file.
|
|
1164
|
+
|
|
1165
|
+
:param path: the file, or the directory containing the files to deploy.
|
|
1166
|
+
:param entrypoint: the main entry point.
|
|
1167
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1168
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1169
|
+
:return: a file-like object containing the bundle tarball.
|
|
1170
|
+
"""
|
|
1171
|
+
|
|
1172
|
+
manifest = create_html_manifest(
|
|
1173
|
+
path=path,
|
|
1174
|
+
entrypoint=entrypoint,
|
|
1175
|
+
extra_files=extra_files,
|
|
1176
|
+
excludes=excludes,
|
|
1177
|
+
)
|
|
1178
|
+
|
|
1179
|
+
if manifest.data.get("files") is None:
|
|
1180
|
+
raise RSConnectException("No valid files were found for the manifest.")
|
|
1181
|
+
if manifest.deploy_dir is None:
|
|
1182
|
+
raise RSConnectException("deploy_dir was not set for the manifest.")
|
|
1183
|
+
|
|
1184
|
+
bundle = Bundle()
|
|
1185
|
+
for f in manifest.data["files"]:
|
|
1186
|
+
if f in manifest.buffer:
|
|
1187
|
+
continue
|
|
1188
|
+
bundle.add_file(f)
|
|
1189
|
+
for k, v in manifest.get_manifest_files_from_buffer().items():
|
|
1190
|
+
bundle.add_to_buffer(k, v)
|
|
1191
|
+
|
|
1192
|
+
manifest_flattened_copy_data = manifest.get_flattened_copy().data
|
|
1193
|
+
bundle.add_to_buffer("manifest.json", json.dumps(manifest_flattened_copy_data, indent=2))
|
|
1194
|
+
|
|
1195
|
+
return bundle.to_file(manifest.deploy_dir)
|
|
1196
|
+
|
|
1197
|
+
|
|
1198
|
+
def make_tensorflow_bundle(
|
|
1199
|
+
directory: str,
|
|
1200
|
+
extra_files: Sequence[str],
|
|
1201
|
+
excludes: Sequence[str],
|
|
1202
|
+
image: Optional[str] = None,
|
|
1203
|
+
) -> typing.IO[bytes]:
|
|
1204
|
+
"""
|
|
1205
|
+
Create an html bundle, given a path and/or entrypoint.
|
|
1206
|
+
|
|
1207
|
+
The bundle contains a manifest.json file created for the given notebook entrypoint file.
|
|
1208
|
+
|
|
1209
|
+
:param directory: the directory containing the TensorFlow model.
|
|
1210
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1211
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1212
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
1213
|
+
:return: a file-like object containing the bundle tarball.
|
|
1214
|
+
"""
|
|
1215
|
+
|
|
1216
|
+
manifest = make_tensorflow_manifest(
|
|
1217
|
+
directory=directory,
|
|
1218
|
+
extra_files=extra_files,
|
|
1219
|
+
excludes=excludes,
|
|
1220
|
+
image=image,
|
|
1221
|
+
)
|
|
1222
|
+
|
|
1223
|
+
if not manifest.get("files"):
|
|
1224
|
+
raise RSConnectException("No valid files were found for the manifest.")
|
|
1225
|
+
|
|
1226
|
+
bundle = Bundle()
|
|
1227
|
+
for f in manifest["files"]:
|
|
1228
|
+
bundle.add_file(join(directory, f))
|
|
1229
|
+
|
|
1230
|
+
bundle.add_to_buffer("manifest.json", json.dumps(manifest, indent=2))
|
|
1231
|
+
|
|
1232
|
+
return bundle.to_file(directory)
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
def create_file_list(
|
|
1236
|
+
path: str,
|
|
1237
|
+
extra_files: Sequence[str],
|
|
1238
|
+
excludes: Sequence[str],
|
|
1239
|
+
use_abspath: bool = False,
|
|
1240
|
+
) -> list[str]:
|
|
1241
|
+
"""
|
|
1242
|
+
Builds a full list of files under the given path that should be included
|
|
1243
|
+
in a manifest or bundle. Extra files and excludes are relative to the given
|
|
1244
|
+
directory and work as you'd expect.
|
|
1245
|
+
|
|
1246
|
+
:param path: a file, or a directory to walk for files.
|
|
1247
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1248
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1249
|
+
:return: the list of relevant files, relative to the given directory.
|
|
1250
|
+
"""
|
|
1251
|
+
extra_files = extra_files or []
|
|
1252
|
+
excludes = excludes if excludes else []
|
|
1253
|
+
glob_set = create_glob_set(path, excludes)
|
|
1254
|
+
exclude_paths = {Path(p) for p in excludes}
|
|
1255
|
+
file_set: set[str] = set(extra_files)
|
|
1256
|
+
|
|
1257
|
+
if isfile(path):
|
|
1258
|
+
path_to_add = abspath(path) if use_abspath else path
|
|
1259
|
+
file_set.add(path_to_add)
|
|
1260
|
+
return sorted(file_set)
|
|
1261
|
+
|
|
1262
|
+
for cur_dir, _, files in os.walk(path):
|
|
1263
|
+
if Path(cur_dir) in exclude_paths:
|
|
1264
|
+
continue
|
|
1265
|
+
if any(parent in exclude_paths for parent in Path(cur_dir).parents):
|
|
1266
|
+
continue
|
|
1267
|
+
for file in files:
|
|
1268
|
+
cur_path = os.path.join(cur_dir, file)
|
|
1269
|
+
rel_path = relpath(cur_path, path)
|
|
1270
|
+
|
|
1271
|
+
if Path(cur_path) in exclude_paths:
|
|
1272
|
+
continue
|
|
1273
|
+
if keep_manifest_specified_file(rel_path, exclude_paths | directories_to_ignore) and (
|
|
1274
|
+
rel_path in extra_files or not glob_set.matches(cur_path)
|
|
1275
|
+
):
|
|
1276
|
+
path_to_add = abspath(cur_path) if use_abspath else rel_path
|
|
1277
|
+
file_set.add(path_to_add)
|
|
1278
|
+
|
|
1279
|
+
return sorted(file_set)
|
|
1280
|
+
|
|
1281
|
+
|
|
1282
|
+
def infer_entrypoint(path: str, mimetype: str) -> str | None:
|
|
1283
|
+
candidates = infer_entrypoint_candidates(path, mimetype)
|
|
1284
|
+
return candidates.pop() if len(candidates) == 1 else None
|
|
1285
|
+
|
|
1286
|
+
|
|
1287
|
+
def infer_entrypoint_candidates(path: str, mimetype: str) -> list[str]:
|
|
1288
|
+
if not path:
|
|
1289
|
+
return []
|
|
1290
|
+
if isfile(path):
|
|
1291
|
+
return [path]
|
|
1292
|
+
if not isdir(path):
|
|
1293
|
+
return []
|
|
1294
|
+
|
|
1295
|
+
default_mimetype_entrypoints: defaultdict[str, str] = defaultdict(str)
|
|
1296
|
+
default_mimetype_entrypoints["text/html"] = "index.html"
|
|
1297
|
+
|
|
1298
|
+
mimetype_filelist: defaultdict[str | None, list[str]] = defaultdict(list)
|
|
1299
|
+
|
|
1300
|
+
for file in os.listdir(path):
|
|
1301
|
+
abs_path = os.path.join(path, file)
|
|
1302
|
+
if not isfile(abs_path):
|
|
1303
|
+
continue
|
|
1304
|
+
file_type = guess_type(file)[0]
|
|
1305
|
+
mimetype_filelist[file_type].append(abs_path)
|
|
1306
|
+
if file in default_mimetype_entrypoints[mimetype]:
|
|
1307
|
+
return [abs_path]
|
|
1308
|
+
return mimetype_filelist[mimetype] or []
|
|
1309
|
+
|
|
1310
|
+
|
|
1311
|
+
def guess_deploy_dir(path: str | Path, entrypoint: Optional[str]) -> str:
|
|
1312
|
+
if path and not exists(path):
|
|
1313
|
+
raise RSConnectException(f"Path {path} does not exist.")
|
|
1314
|
+
# The entrypoint is a bare basename meant to be resolved relative to ``path``
|
|
1315
|
+
# (the later logic does ``join(abs_path, basename(entrypoint))``). Accept it
|
|
1316
|
+
# when it exists relative to the CWD or inside ``path``; only reject when neither.
|
|
1317
|
+
if entrypoint and not exists(entrypoint):
|
|
1318
|
+
if not (path and isfile(os.path.join(abspath(path), basename(entrypoint)))):
|
|
1319
|
+
raise RSConnectException(f"Entrypoint {entrypoint} does not exist.")
|
|
1320
|
+
abs_path = abspath(path)
|
|
1321
|
+
abs_entrypoint = abspath(entrypoint) if entrypoint else None
|
|
1322
|
+
if not path and not entrypoint:
|
|
1323
|
+
raise RSConnectException("No path or entrypoint provided.")
|
|
1324
|
+
deploy_dir: str
|
|
1325
|
+
if path and isfile(path):
|
|
1326
|
+
if not entrypoint:
|
|
1327
|
+
deploy_dir = dirname(abs_path)
|
|
1328
|
+
elif isfile(entrypoint):
|
|
1329
|
+
if abs_path == abs_entrypoint:
|
|
1330
|
+
deploy_dir = dirname(abs_path)
|
|
1331
|
+
else:
|
|
1332
|
+
raise RSConnectException("Path and entrypoint need to match if they are both files.")
|
|
1333
|
+
elif isdir(entrypoint):
|
|
1334
|
+
raise RSConnectException("Entrypoint cannot be a directory while the path is a file.")
|
|
1335
|
+
else:
|
|
1336
|
+
raise RSConnectException("Entrypoint cannot be a special file.")
|
|
1337
|
+
|
|
1338
|
+
elif path and isdir(path):
|
|
1339
|
+
if not entrypoint or not abs_entrypoint:
|
|
1340
|
+
deploy_dir = abs_path
|
|
1341
|
+
# elif entrypoint and isdir(entrypoint):
|
|
1342
|
+
# raise RSConnectException("Path and entrypoint cannot both be directories.")
|
|
1343
|
+
else:
|
|
1344
|
+
if isdir(entrypoint):
|
|
1345
|
+
raise RSConnectException("Path and entrypoint cannot both be directories.")
|
|
1346
|
+
guess_entry_file = os.path.join(abs_path, basename(entrypoint))
|
|
1347
|
+
if isfile(guess_entry_file):
|
|
1348
|
+
deploy_dir = dirname(guess_entry_file)
|
|
1349
|
+
elif isfile(entrypoint):
|
|
1350
|
+
deploy_dir = dirname(abs_entrypoint)
|
|
1351
|
+
else:
|
|
1352
|
+
raise RSConnectException("Can't find entrypoint.")
|
|
1353
|
+
elif not path and entrypoint:
|
|
1354
|
+
raise RSConnectException("A path needs to be provided.")
|
|
1355
|
+
else:
|
|
1356
|
+
deploy_dir = abs_path
|
|
1357
|
+
return deploy_dir
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def resolve_shiny_express_entrypoint(entrypoint: str, directory: str) -> str:
|
|
1361
|
+
"""Rewrite a Shiny entrypoint to its Shiny Express module form when needed.
|
|
1362
|
+
|
|
1363
|
+
Connect runs Shiny Express apps through the ``shiny.express.app:<var>``
|
|
1364
|
+
module rather than a plain file, so both ``deploy shiny`` and
|
|
1365
|
+
``deploy pyproject`` must apply this rewrite to deploy a working app.
|
|
1366
|
+
|
|
1367
|
+
Accepts a bare module name (``"app"``) or a filename (``"app.py"``).
|
|
1368
|
+
Returns the ``shiny.express.app:<escaped>`` form when the app file is a
|
|
1369
|
+
Shiny Express app, otherwise returns ``entrypoint`` unchanged.
|
|
1370
|
+
|
|
1371
|
+
:param str entrypoint: the configured entrypoint, with or without ``.py``.
|
|
1372
|
+
:param str directory: directory containing the app file.
|
|
1373
|
+
"""
|
|
1374
|
+
app_file = entrypoint if entrypoint.lower().endswith(".py") else entrypoint + ".py"
|
|
1375
|
+
if is_express_app(app_file, directory):
|
|
1376
|
+
return "shiny.express.app:" + escape_to_var_name(app_file)
|
|
1377
|
+
return entrypoint
|
|
1378
|
+
|
|
1379
|
+
|
|
1380
|
+
def abs_entrypoint(path: str | Path, entrypoint: str) -> str | None:
|
|
1381
|
+
if isfile(entrypoint):
|
|
1382
|
+
return abspath(entrypoint)
|
|
1383
|
+
guess_entry_file = os.path.join(abspath(path), basename(entrypoint))
|
|
1384
|
+
if isfile(guess_entry_file):
|
|
1385
|
+
return guess_entry_file
|
|
1386
|
+
return None
|
|
1387
|
+
|
|
1388
|
+
|
|
1389
|
+
def make_voila_bundle(
|
|
1390
|
+
path: str,
|
|
1391
|
+
entrypoint: Optional[str],
|
|
1392
|
+
extra_files: Sequence[str],
|
|
1393
|
+
excludes: Sequence[str],
|
|
1394
|
+
force_generate: bool,
|
|
1395
|
+
environment: Environment,
|
|
1396
|
+
image: Optional[str] = None,
|
|
1397
|
+
env_management_py: Optional[bool] = None,
|
|
1398
|
+
env_management_r: Optional[bool] = None,
|
|
1399
|
+
r_environment: Optional[REnvironment] = None,
|
|
1400
|
+
multi_notebook: bool = False,
|
|
1401
|
+
) -> typing.IO[bytes]:
|
|
1402
|
+
"""
|
|
1403
|
+
Create an voila bundle, given a path and/or entrypoint.
|
|
1404
|
+
|
|
1405
|
+
The bundle contains a manifest.json file created for the given notebook entrypoint file.
|
|
1406
|
+
If the related environment file (requirements.txt) doesn't
|
|
1407
|
+
exist (or force_generate is set to True), the environment file will also be written.
|
|
1408
|
+
|
|
1409
|
+
:param path: the file, or the directory containing the files to deploy.
|
|
1410
|
+
:param entrypoint: the main entry point.
|
|
1411
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1412
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1413
|
+
:param force_generate: bool indicating whether to force generate manifest and related environment files.
|
|
1414
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
1415
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
1416
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1417
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
1418
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1419
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
1420
|
+
:return: a file-like object containing the bundle tarball.
|
|
1421
|
+
"""
|
|
1422
|
+
|
|
1423
|
+
manifest = create_voila_manifest(
|
|
1424
|
+
path=path,
|
|
1425
|
+
entrypoint=entrypoint,
|
|
1426
|
+
extra_files=extra_files,
|
|
1427
|
+
excludes=excludes,
|
|
1428
|
+
force_generate=force_generate,
|
|
1429
|
+
environment=environment,
|
|
1430
|
+
image=image,
|
|
1431
|
+
env_management_py=env_management_py,
|
|
1432
|
+
env_management_r=env_management_r,
|
|
1433
|
+
r_environment=r_environment,
|
|
1434
|
+
multi_notebook=multi_notebook,
|
|
1435
|
+
)
|
|
1436
|
+
|
|
1437
|
+
if manifest.data.get("files") is None:
|
|
1438
|
+
raise RSConnectException("No valid files were found for the manifest.")
|
|
1439
|
+
if manifest.deploy_dir is None:
|
|
1440
|
+
raise RSConnectException("deploy_dir was not set for the manifest.")
|
|
1441
|
+
|
|
1442
|
+
bundle = Bundle()
|
|
1443
|
+
for f in manifest.data["files"]:
|
|
1444
|
+
if f in manifest.buffer:
|
|
1445
|
+
continue
|
|
1446
|
+
bundle.add_file(f)
|
|
1447
|
+
for k, v in manifest.get_manifest_files_from_buffer().items():
|
|
1448
|
+
bundle.add_to_buffer(k, v)
|
|
1449
|
+
|
|
1450
|
+
manifest_flattened_copy_data = manifest.get_flattened_copy().data
|
|
1451
|
+
if multi_notebook and "metadata" in manifest_flattened_copy_data:
|
|
1452
|
+
manifest_flattened_copy_data["metadata"]["entrypoint"] = ""
|
|
1453
|
+
bundle.add_to_buffer("manifest.json", json.dumps(manifest_flattened_copy_data, indent=2))
|
|
1454
|
+
|
|
1455
|
+
return bundle.to_file(manifest.deploy_dir)
|
|
1456
|
+
|
|
1457
|
+
|
|
1458
|
+
def make_api_bundle(
|
|
1459
|
+
directory: str,
|
|
1460
|
+
entry_point: str,
|
|
1461
|
+
app_mode: AppMode,
|
|
1462
|
+
environment: Environment,
|
|
1463
|
+
extra_files: Sequence[str],
|
|
1464
|
+
excludes: Sequence[str],
|
|
1465
|
+
image: Optional[str] = None,
|
|
1466
|
+
env_management_py: Optional[bool] = None,
|
|
1467
|
+
env_management_r: Optional[bool] = None,
|
|
1468
|
+
r_environment: Optional[REnvironment] = None,
|
|
1469
|
+
) -> typing.IO[bytes]:
|
|
1470
|
+
"""
|
|
1471
|
+
Create an API bundle, given a directory path and a manifest.
|
|
1472
|
+
|
|
1473
|
+
:param directory: the directory containing the files to deploy.
|
|
1474
|
+
:param entry_point: the main entry point for the API.
|
|
1475
|
+
:param app_mode: the app mode to use.
|
|
1476
|
+
:param environment: the Python environment information.
|
|
1477
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1478
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1479
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
1480
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
1481
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1482
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
1483
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1484
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
1485
|
+
:return: a file-like object containing the bundle tarball.
|
|
1486
|
+
"""
|
|
1487
|
+
manifest, relevant_files = make_api_manifest(
|
|
1488
|
+
directory,
|
|
1489
|
+
entry_point,
|
|
1490
|
+
app_mode,
|
|
1491
|
+
environment,
|
|
1492
|
+
extra_files,
|
|
1493
|
+
excludes,
|
|
1494
|
+
image,
|
|
1495
|
+
env_management_py,
|
|
1496
|
+
env_management_r,
|
|
1497
|
+
r_environment,
|
|
1498
|
+
)
|
|
1499
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
1500
|
+
|
|
1501
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
1502
|
+
bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2))
|
|
1503
|
+
bundle_add_buffer(bundle, environment.filename, environment.contents)
|
|
1504
|
+
|
|
1505
|
+
for rel_path in relevant_files:
|
|
1506
|
+
bundle_add_file(bundle, rel_path, directory)
|
|
1507
|
+
|
|
1508
|
+
# rewind file pointer
|
|
1509
|
+
bundle_file.seek(0)
|
|
1510
|
+
|
|
1511
|
+
return bundle_file
|
|
1512
|
+
|
|
1513
|
+
|
|
1514
|
+
def make_nodejs_manifest(
|
|
1515
|
+
directory: str,
|
|
1516
|
+
entry_point: str,
|
|
1517
|
+
node_environment: NodeEnvironment,
|
|
1518
|
+
extra_files: Sequence[str],
|
|
1519
|
+
excludes: Sequence[str],
|
|
1520
|
+
image: Optional[str] = None,
|
|
1521
|
+
env_management_node: Optional[bool] = None,
|
|
1522
|
+
) -> tuple[ManifestData, list[str]]:
|
|
1523
|
+
"""
|
|
1524
|
+
Makes a manifest for a Node.js application.
|
|
1525
|
+
|
|
1526
|
+
:param directory: the directory containing the files to deploy.
|
|
1527
|
+
:param entry_point: the main entry point file (e.g., "app.js").
|
|
1528
|
+
:param node_environment: the Node.js environment information.
|
|
1529
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1530
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1531
|
+
:param image: optional docker image for off-host execution.
|
|
1532
|
+
:param env_management_node: False prevents Connect from managing the Node.js environment.
|
|
1533
|
+
:return: the manifest and a list of the files involved.
|
|
1534
|
+
"""
|
|
1535
|
+
extra_files = list(extra_files or [])
|
|
1536
|
+
skip = ["manifest.json"]
|
|
1537
|
+
extra_files = sorted(list(set(extra_files) - set(skip)))
|
|
1538
|
+
|
|
1539
|
+
excludes = list(excludes) if excludes else []
|
|
1540
|
+
excludes.append("manifest.json")
|
|
1541
|
+
excludes.append("node_modules")
|
|
1542
|
+
excludes.extend(list_environment_dirs(directory))
|
|
1543
|
+
|
|
1544
|
+
relevant_files = create_file_list(directory, extra_files, excludes)
|
|
1545
|
+
|
|
1546
|
+
manifest: ManifestData = {
|
|
1547
|
+
"version": 1,
|
|
1548
|
+
"metadata": {
|
|
1549
|
+
"appmode": AppModes.NODE_JS.name(),
|
|
1550
|
+
"entrypoint": entry_point,
|
|
1551
|
+
},
|
|
1552
|
+
"node": {
|
|
1553
|
+
"version": node_environment.node_version,
|
|
1554
|
+
"package_manager": {
|
|
1555
|
+
"name": "npm",
|
|
1556
|
+
"version": node_environment.npm_version,
|
|
1557
|
+
"package_file": node_environment.package_file,
|
|
1558
|
+
},
|
|
1559
|
+
},
|
|
1560
|
+
"files": {},
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
if node_environment.locale:
|
|
1564
|
+
manifest["locale"] = node_environment.locale
|
|
1565
|
+
|
|
1566
|
+
if image or env_management_node is not None:
|
|
1567
|
+
manifest_environment: ManifestDataEnvironment = {}
|
|
1568
|
+
if image:
|
|
1569
|
+
manifest_environment["image"] = image
|
|
1570
|
+
if env_management_node is not None:
|
|
1571
|
+
manifest_environment["environment_management"] = {"node": env_management_node}
|
|
1572
|
+
manifest["environment"] = manifest_environment
|
|
1573
|
+
|
|
1574
|
+
for rel_path in relevant_files:
|
|
1575
|
+
manifest_add_file(manifest, rel_path, directory)
|
|
1576
|
+
|
|
1577
|
+
return manifest, relevant_files
|
|
1578
|
+
|
|
1579
|
+
|
|
1580
|
+
def make_nodejs_bundle(
|
|
1581
|
+
directory: str,
|
|
1582
|
+
entry_point: str,
|
|
1583
|
+
node_environment: NodeEnvironment,
|
|
1584
|
+
extra_files: Sequence[str],
|
|
1585
|
+
excludes: Sequence[str],
|
|
1586
|
+
image: Optional[str] = None,
|
|
1587
|
+
env_management_node: Optional[bool] = None,
|
|
1588
|
+
) -> typing.IO[bytes]:
|
|
1589
|
+
"""
|
|
1590
|
+
Create a Node.js application bundle, given a directory path.
|
|
1591
|
+
|
|
1592
|
+
:param directory: the directory containing the files to deploy.
|
|
1593
|
+
:param entry_point: the main entry point file (e.g., "app.js").
|
|
1594
|
+
:param node_environment: the Node.js environment information.
|
|
1595
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1596
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1597
|
+
:param image: optional docker image for off-host execution.
|
|
1598
|
+
:param env_management_node: False prevents Connect from managing the Node.js environment.
|
|
1599
|
+
:return: a file-like object containing the bundle tarball.
|
|
1600
|
+
"""
|
|
1601
|
+
manifest, relevant_files = make_nodejs_manifest(
|
|
1602
|
+
directory,
|
|
1603
|
+
entry_point,
|
|
1604
|
+
node_environment,
|
|
1605
|
+
extra_files,
|
|
1606
|
+
excludes,
|
|
1607
|
+
image,
|
|
1608
|
+
env_management_node,
|
|
1609
|
+
)
|
|
1610
|
+
bundle_file = tempfile.TemporaryFile(prefix="rsc_bundle")
|
|
1611
|
+
|
|
1612
|
+
with tarfile.open(mode="w:gz", fileobj=bundle_file) as bundle:
|
|
1613
|
+
bundle_add_buffer(bundle, "manifest.json", json.dumps(manifest, indent=2))
|
|
1614
|
+
|
|
1615
|
+
for rel_path in relevant_files:
|
|
1616
|
+
bundle_add_file(bundle, rel_path, directory)
|
|
1617
|
+
|
|
1618
|
+
bundle_file.seek(0)
|
|
1619
|
+
|
|
1620
|
+
return bundle_file
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def _create_quarto_file_list(
|
|
1624
|
+
directory: str,
|
|
1625
|
+
extra_files: Sequence[str],
|
|
1626
|
+
excludes: Sequence[str],
|
|
1627
|
+
) -> list[str]:
|
|
1628
|
+
"""
|
|
1629
|
+
Builds a full list of files under the given directory that should be included
|
|
1630
|
+
in a manifest or bundle. Extra files and excludes are relative to the given
|
|
1631
|
+
directory and work as you'd expect.
|
|
1632
|
+
|
|
1633
|
+
:param directory: the directory to walk for files.
|
|
1634
|
+
:param extra_files: a sequence of any extra files to include in the bundle.
|
|
1635
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
1636
|
+
:return: the list of relevant files, relative to the given directory.
|
|
1637
|
+
"""
|
|
1638
|
+
# Don't let these top-level files be added via the extra files list.
|
|
1639
|
+
extra_files = extra_files or []
|
|
1640
|
+
skip = ["manifest.json"]
|
|
1641
|
+
extra_files = sorted(list(set(extra_files) - set(skip)))
|
|
1642
|
+
|
|
1643
|
+
# Don't include these top-level files.
|
|
1644
|
+
excludes = list(excludes) if excludes else []
|
|
1645
|
+
excludes.append("manifest.json")
|
|
1646
|
+
excludes.extend(list_environment_dirs(directory))
|
|
1647
|
+
|
|
1648
|
+
file_list = create_file_list(directory, extra_files, excludes)
|
|
1649
|
+
return file_list
|
|
1650
|
+
|
|
1651
|
+
|
|
1652
|
+
def make_quarto_manifest(
|
|
1653
|
+
file_or_directory: str,
|
|
1654
|
+
quarto_inspection: QuartoInspectResult,
|
|
1655
|
+
app_mode: AppMode,
|
|
1656
|
+
environment: Optional[Environment],
|
|
1657
|
+
extra_files: Sequence[str],
|
|
1658
|
+
excludes: Sequence[str],
|
|
1659
|
+
image: Optional[str] = None,
|
|
1660
|
+
env_management_py: Optional[bool] = None,
|
|
1661
|
+
env_management_r: Optional[bool] = None,
|
|
1662
|
+
r_environment: Optional[REnvironment] = None,
|
|
1663
|
+
) -> tuple[ManifestData, list[str]]:
|
|
1664
|
+
"""
|
|
1665
|
+
Makes a manifest for a Quarto project.
|
|
1666
|
+
|
|
1667
|
+
:param file_or_directory: The Quarto document or the directory containing the Quarto project.
|
|
1668
|
+
:param quarto_inspection: The parsed JSON from a 'quarto inspect' against the project.
|
|
1669
|
+
:param app_mode: The application mode to assume.
|
|
1670
|
+
:param environment: The (optional) Python environment to use.
|
|
1671
|
+
:param extra_files: Any extra files to include in the manifest.
|
|
1672
|
+
:param excludes: A sequence of glob patterns to exclude when enumerating files to bundle.
|
|
1673
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
1674
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
1675
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1676
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
1677
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1678
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
1679
|
+
:return: the manifest and a list of the files involved.
|
|
1680
|
+
"""
|
|
1681
|
+
if environment:
|
|
1682
|
+
extra_files = list(extra_files or [])
|
|
1683
|
+
|
|
1684
|
+
base_dir = file_or_directory
|
|
1685
|
+
if isdir(file_or_directory):
|
|
1686
|
+
# Directory as a Quarto project.
|
|
1687
|
+
excludes = list(excludes or []) + [".quarto"]
|
|
1688
|
+
|
|
1689
|
+
project_config = quarto_inspection.get("config", {}).get("project", {})
|
|
1690
|
+
output_dir = cast(Union[str, None], project_config.get("output-dir", None))
|
|
1691
|
+
if output_dir:
|
|
1692
|
+
excludes = excludes + [output_dir]
|
|
1693
|
+
|
|
1694
|
+
files_data = quarto_inspection.get("files", {})
|
|
1695
|
+
files_input_data = files_data.get("input", [])
|
|
1696
|
+
# files.input is a list of absolute paths to input (rendered)
|
|
1697
|
+
# files. Automatically ignore the most common derived files for
|
|
1698
|
+
# those inputs.
|
|
1699
|
+
#
|
|
1700
|
+
# These files are ignored even when the project has an output
|
|
1701
|
+
# directory, as Quarto may create these files while a render is
|
|
1702
|
+
# in-flight.
|
|
1703
|
+
for each in files_input_data:
|
|
1704
|
+
t, _ = splitext(os.path.relpath(each, file_or_directory))
|
|
1705
|
+
excludes = excludes + [t + ".html", t + "_files/**/*"]
|
|
1706
|
+
|
|
1707
|
+
# relevant files don't need to include requirements.txt file because it is
|
|
1708
|
+
# always added to the manifest (as a buffer) from the environment contents
|
|
1709
|
+
if environment:
|
|
1710
|
+
excludes.append(environment.filename)
|
|
1711
|
+
|
|
1712
|
+
relevant_files = _create_quarto_file_list(base_dir, extra_files, excludes)
|
|
1713
|
+
else:
|
|
1714
|
+
# Standalone Quarto document
|
|
1715
|
+
base_dir = dirname(file_or_directory)
|
|
1716
|
+
file_name = basename(file_or_directory)
|
|
1717
|
+
relevant_files = [file_name] + list(extra_files or [])
|
|
1718
|
+
|
|
1719
|
+
manifest = make_source_manifest(
|
|
1720
|
+
app_mode,
|
|
1721
|
+
environment,
|
|
1722
|
+
None,
|
|
1723
|
+
quarto_inspection,
|
|
1724
|
+
image,
|
|
1725
|
+
env_management_py,
|
|
1726
|
+
env_management_r,
|
|
1727
|
+
r_environment,
|
|
1728
|
+
)
|
|
1729
|
+
|
|
1730
|
+
if environment:
|
|
1731
|
+
manifest_add_buffer(manifest, environment.filename, environment.contents)
|
|
1732
|
+
|
|
1733
|
+
for rel_path in relevant_files:
|
|
1734
|
+
manifest_add_file(manifest, rel_path, base_dir)
|
|
1735
|
+
|
|
1736
|
+
return manifest, relevant_files
|
|
1737
|
+
|
|
1738
|
+
|
|
1739
|
+
def _enforce_title_length(title: str) -> str:
|
|
1740
|
+
"""
|
|
1741
|
+
Pad or truncate a title so it is between 3 and 1024 characters long, as
|
|
1742
|
+
required by Posit Connect.
|
|
1743
|
+
"""
|
|
1744
|
+
return title[:1024].rjust(3, "0")
|
|
1745
|
+
|
|
1746
|
+
|
|
1747
|
+
def _default_title(file_name: str | Path) -> str:
|
|
1748
|
+
"""
|
|
1749
|
+
Produce a default content title from the given file path. The result is
|
|
1750
|
+
guaranteed to be between 3 and 1024 characters long, as required by Posit
|
|
1751
|
+
Connect.
|
|
1752
|
+
|
|
1753
|
+
:param file_name: the name from which the title will be derived.
|
|
1754
|
+
:return: the derived title.
|
|
1755
|
+
"""
|
|
1756
|
+
# Make sure we have enough of a path to derive text from.
|
|
1757
|
+
file_name = abspath(file_name)
|
|
1758
|
+
# noinspection PyTypeChecker
|
|
1759
|
+
return _enforce_title_length(basename(file_name).rsplit(".", 1)[0])
|
|
1760
|
+
|
|
1761
|
+
|
|
1762
|
+
def validate_file_is_notebook(file_name: str | Path) -> None:
|
|
1763
|
+
"""
|
|
1764
|
+
Validate that the given file is a Jupyter Notebook. If it isn't, an exception is
|
|
1765
|
+
thrown. A file must exist and have the '.ipynb' extension.
|
|
1766
|
+
|
|
1767
|
+
:param file_name: the name of the file to validate.
|
|
1768
|
+
"""
|
|
1769
|
+
file_suffix = splitext(file_name)[1].lower()
|
|
1770
|
+
if file_suffix != ".ipynb" or not exists(file_name):
|
|
1771
|
+
raise RSConnectException("A Jupyter notebook (.ipynb) file is required here.")
|
|
1772
|
+
|
|
1773
|
+
|
|
1774
|
+
def validate_extra_files(
|
|
1775
|
+
directory: str | Path,
|
|
1776
|
+
extra_files: Sequence[str] | None,
|
|
1777
|
+
use_abspath: bool = False,
|
|
1778
|
+
) -> list[str]:
|
|
1779
|
+
"""
|
|
1780
|
+
If the user specified a list of extra files, validate that they all exist and are
|
|
1781
|
+
beneath the given directory and, if so, return a list of them made relative to that
|
|
1782
|
+
directory.
|
|
1783
|
+
|
|
1784
|
+
:param directory: the directory that the extra files must be relative to.
|
|
1785
|
+
:param extra_files: the list of extra files to qualify and validate.
|
|
1786
|
+
:return: the extra files qualified by the directory.
|
|
1787
|
+
"""
|
|
1788
|
+
result: list[str] = []
|
|
1789
|
+
if extra_files:
|
|
1790
|
+
for extra in extra_files:
|
|
1791
|
+
extra_file = relpath(extra, directory)
|
|
1792
|
+
# It's an error if we have to leave the given dir to get to the extra
|
|
1793
|
+
# file.
|
|
1794
|
+
if extra_file.startswith("../"):
|
|
1795
|
+
raise RSConnectException("%s must be under %s." % (extra_file, directory))
|
|
1796
|
+
if not exists(join(directory, extra_file)):
|
|
1797
|
+
raise RSConnectException("Could not find file %s under %s" % (extra, directory))
|
|
1798
|
+
extra_file = abspath(join(directory, extra_file)) if use_abspath else extra_file
|
|
1799
|
+
result.append(extra_file)
|
|
1800
|
+
return result
|
|
1801
|
+
|
|
1802
|
+
|
|
1803
|
+
def validate_manifest_file(file_or_directory: str) -> str:
|
|
1804
|
+
"""
|
|
1805
|
+
Validates that the name given represents either an existing manifest.json file or
|
|
1806
|
+
a directory that contains one. If not, an exception is raised.
|
|
1807
|
+
|
|
1808
|
+
:param file_or_directory: the name of the manifest file or directory that contains it.
|
|
1809
|
+
:return: the real path to the manifest file.
|
|
1810
|
+
"""
|
|
1811
|
+
if isdir(file_or_directory):
|
|
1812
|
+
file_or_directory = join(file_or_directory, "manifest.json")
|
|
1813
|
+
if basename(file_or_directory) != "manifest.json" or not exists(file_or_directory):
|
|
1814
|
+
raise RSConnectException("A manifest.json file or a directory containing one is required here.")
|
|
1815
|
+
return file_or_directory
|
|
1816
|
+
|
|
1817
|
+
|
|
1818
|
+
re_app_prefix = re.compile(r"^app[-_].+\.py$")
|
|
1819
|
+
re_app_suffix = re.compile(r".+[-_]app\.py$")
|
|
1820
|
+
|
|
1821
|
+
|
|
1822
|
+
def get_default_entrypoint(directory: str | Path) -> str:
|
|
1823
|
+
candidates = ["app", "application", "main", "api"]
|
|
1824
|
+
files = set(os.listdir(directory))
|
|
1825
|
+
|
|
1826
|
+
for candidate in candidates:
|
|
1827
|
+
filename = candidate + ".py"
|
|
1828
|
+
if filename in files:
|
|
1829
|
+
return candidate
|
|
1830
|
+
|
|
1831
|
+
# if only one python source file, use it
|
|
1832
|
+
python_files = list(filter(lambda s: s.endswith(".py"), files))
|
|
1833
|
+
if len(python_files) == 1:
|
|
1834
|
+
return python_files[0][:-3]
|
|
1835
|
+
|
|
1836
|
+
# try app-*.py, app_*.py, *-app.py, *_app.py
|
|
1837
|
+
app_files = list(filter(lambda s: re_app_prefix.match(s) or re_app_suffix.match(s), python_files))
|
|
1838
|
+
if len(app_files) == 1:
|
|
1839
|
+
# In these cases, the app should be in the "app" attribute
|
|
1840
|
+
return app_files[0][:-3]
|
|
1841
|
+
|
|
1842
|
+
raise RSConnectException(f"Could not determine default entrypoint file in directory '{directory}'")
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
def validate_entry_point(entry_point: str | None, directory: str) -> str:
|
|
1846
|
+
"""
|
|
1847
|
+
Validates the entry point specified by the user, expanding as necessary. If the
|
|
1848
|
+
user specifies nothing, a module of "app" is assumed. If the user specifies a
|
|
1849
|
+
module only, at runtime the following object names will be tried in order: `app`,
|
|
1850
|
+
`application`, `create_app`, and `make_app`.
|
|
1851
|
+
|
|
1852
|
+
:param entry_point: the entry point as specified by the user.
|
|
1853
|
+
:return: An entry point, in the form of "module" or "module:app".
|
|
1854
|
+
"""
|
|
1855
|
+
if not entry_point:
|
|
1856
|
+
entry_point = get_default_entrypoint(directory)
|
|
1857
|
+
|
|
1858
|
+
parts = entry_point.split(":")
|
|
1859
|
+
|
|
1860
|
+
if len(parts) > 2:
|
|
1861
|
+
raise RSConnectException('Entry point is not in "module:object" format.')
|
|
1862
|
+
|
|
1863
|
+
return entry_point
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
def get_default_node_entrypoint(directory: str | Path) -> str:
|
|
1867
|
+
"""
|
|
1868
|
+
Determine the default entry point for a Node.js application.
|
|
1869
|
+
|
|
1870
|
+
Checks package.json "main" field first, then falls back to common filenames.
|
|
1871
|
+
|
|
1872
|
+
:param directory: the directory containing the Node.js application.
|
|
1873
|
+
:return: the entry point filename (e.g., "app.js").
|
|
1874
|
+
"""
|
|
1875
|
+
package_json_path = join(str(directory), "package.json")
|
|
1876
|
+
if isfile(package_json_path):
|
|
1877
|
+
with open(package_json_path, encoding="utf-8") as f:
|
|
1878
|
+
try:
|
|
1879
|
+
package_data = json.load(f)
|
|
1880
|
+
except json.JSONDecodeError:
|
|
1881
|
+
package_data = {}
|
|
1882
|
+
|
|
1883
|
+
# Check "main" field
|
|
1884
|
+
main = package_data.get("main")
|
|
1885
|
+
if main and isfile(join(str(directory), main)):
|
|
1886
|
+
return main
|
|
1887
|
+
|
|
1888
|
+
# Check "scripts.start" for "node <file>" pattern
|
|
1889
|
+
start_script = (package_data.get("scripts") or {}).get("start", "")
|
|
1890
|
+
match = re.match(r"node\s+(\S+)", start_script)
|
|
1891
|
+
if match:
|
|
1892
|
+
start_file = match.group(1)
|
|
1893
|
+
if isfile(join(str(directory), start_file)):
|
|
1894
|
+
return start_file
|
|
1895
|
+
|
|
1896
|
+
# Fall back to common filenames
|
|
1897
|
+
files = set(os.listdir(directory))
|
|
1898
|
+
for candidate in ["app.js", "index.js", "server.js", "main.js", "app.ts", "index.ts", "server.ts", "main.ts"]:
|
|
1899
|
+
if candidate in files:
|
|
1900
|
+
return candidate
|
|
1901
|
+
|
|
1902
|
+
raise RSConnectException(f"Could not determine default entrypoint file in directory '{directory}'")
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
def validate_node_entry_point(entry_point: str | None, directory: str) -> str:
|
|
1906
|
+
"""
|
|
1907
|
+
Validates the entry point for a Node.js application.
|
|
1908
|
+
|
|
1909
|
+
If no entry point is specified, auto-detects from package.json or common filenames.
|
|
1910
|
+
Validates that the entry point file exists in the directory.
|
|
1911
|
+
|
|
1912
|
+
:param entry_point: the entry point as specified by the user, or None for auto-detection.
|
|
1913
|
+
:param directory: the directory containing the Node.js application.
|
|
1914
|
+
:return: the validated entry point filename.
|
|
1915
|
+
"""
|
|
1916
|
+
if not entry_point:
|
|
1917
|
+
entry_point = get_default_node_entrypoint(directory)
|
|
1918
|
+
|
|
1919
|
+
entry_path = join(directory, entry_point)
|
|
1920
|
+
if not isfile(entry_path):
|
|
1921
|
+
raise RSConnectException(f"The entry point file '{entry_point}' does not exist in '{directory}'.")
|
|
1922
|
+
|
|
1923
|
+
return entry_point
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
def _warn_on_ignored_entrypoint(entrypoint: Optional[str]) -> None:
|
|
1927
|
+
if entrypoint:
|
|
1928
|
+
click.secho(
|
|
1929
|
+
" Warning: entrypoint will not be used or considered for multi-notebook mode.",
|
|
1930
|
+
fg="yellow",
|
|
1931
|
+
)
|
|
1932
|
+
|
|
1933
|
+
|
|
1934
|
+
def create_notebook_manifest_and_environment_file(
|
|
1935
|
+
entry_point_file: str,
|
|
1936
|
+
environment: Environment,
|
|
1937
|
+
app_mode: AppMode,
|
|
1938
|
+
extra_files: Sequence[str],
|
|
1939
|
+
force: bool,
|
|
1940
|
+
hide_all_input: bool,
|
|
1941
|
+
hide_tagged_input: bool,
|
|
1942
|
+
image: Optional[str] = None,
|
|
1943
|
+
env_management_py: Optional[bool] = None,
|
|
1944
|
+
env_management_r: Optional[bool] = None,
|
|
1945
|
+
) -> None:
|
|
1946
|
+
"""
|
|
1947
|
+
Creates and writes a manifest.json file for the given notebook entry point file.
|
|
1948
|
+
If the related environment file (requirements.txt, environment.yml, etc.) doesn't
|
|
1949
|
+
exist (or force is set to True), the environment file will also be written.
|
|
1950
|
+
|
|
1951
|
+
:param entry_point_file: the entry point file (Jupyter notebook, etc.) to build
|
|
1952
|
+
the manifest for.
|
|
1953
|
+
:param environment: the Python environment to start with. This should be what's
|
|
1954
|
+
returned by the inspect_environment() function.
|
|
1955
|
+
:param app_mode: the application mode to assume. If this is None, the extension
|
|
1956
|
+
portion of the entry point file name will be used to derive one. Previous default = None.
|
|
1957
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
1958
|
+
:param force: if True, forces the environment file to be written. even if it
|
|
1959
|
+
already exists. Previous default = True.
|
|
1960
|
+
:param hide_all_input: if True, will hide all input cells when rendering output. Previous default = False.
|
|
1961
|
+
:param hide_tagged_input: If True, will hide input code cells with the 'hide_input' tag
|
|
1962
|
+
when rendering output. Previous default = False.
|
|
1963
|
+
:param image: an optional docker image for off-host execution. Previous default = None.
|
|
1964
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
1965
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1966
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
1967
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
1968
|
+
:return:
|
|
1969
|
+
"""
|
|
1970
|
+
if (
|
|
1971
|
+
not write_notebook_manifest_json(
|
|
1972
|
+
entry_point_file,
|
|
1973
|
+
environment,
|
|
1974
|
+
app_mode,
|
|
1975
|
+
extra_files,
|
|
1976
|
+
hide_all_input,
|
|
1977
|
+
hide_tagged_input,
|
|
1978
|
+
image,
|
|
1979
|
+
env_management_py,
|
|
1980
|
+
env_management_r,
|
|
1981
|
+
)
|
|
1982
|
+
or force
|
|
1983
|
+
):
|
|
1984
|
+
write_environment_file(environment, dirname(entry_point_file))
|
|
1985
|
+
|
|
1986
|
+
|
|
1987
|
+
def write_notebook_manifest_json(
|
|
1988
|
+
entry_point_file: str,
|
|
1989
|
+
environment: Environment,
|
|
1990
|
+
app_mode: AppMode,
|
|
1991
|
+
extra_files: Sequence[str],
|
|
1992
|
+
hide_all_input: Optional[bool],
|
|
1993
|
+
hide_tagged_input: Optional[bool],
|
|
1994
|
+
image: Optional[str] = None,
|
|
1995
|
+
env_management_py: Optional[bool] = None,
|
|
1996
|
+
env_management_r: Optional[bool] = None,
|
|
1997
|
+
r_environment: Optional[REnvironment] = None,
|
|
1998
|
+
) -> bool:
|
|
1999
|
+
"""
|
|
2000
|
+
Creates and writes a manifest.json file for the given entry point file. If
|
|
2001
|
+
the application mode is not provided, an attempt will be made to resolve one
|
|
2002
|
+
based on the extension portion of the entry point file.
|
|
2003
|
+
|
|
2004
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
2005
|
+
:param entry_point_file: the entry point file (Jupyter notebook, etc.) to build
|
|
2006
|
+
the manifest for.
|
|
2007
|
+
:param environment: the Python environment to start with. This should be what's
|
|
2008
|
+
returned by the inspect_environment() function.
|
|
2009
|
+
:param app_mode: the application mode to assume. If this is None, the extension
|
|
2010
|
+
portion of the entry point file name will be used to derive one. Previous default = None.
|
|
2011
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
2012
|
+
:param hide_all_input: if True, will hide all input cells when rendering output. Previous default = False.
|
|
2013
|
+
:param hide_tagged_input: If True, will hide input code cells with the 'hide_input' tag
|
|
2014
|
+
when rendering output. Previous default = False.
|
|
2015
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2016
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
2017
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2018
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
2019
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2020
|
+
:return: whether or not the environment file (requirements.txt, environment.yml,
|
|
2021
|
+
etc.) that goes along with the manifest exists.
|
|
2022
|
+
"""
|
|
2023
|
+
extra_files = validate_extra_files(dirname(entry_point_file), extra_files)
|
|
2024
|
+
directory = dirname(entry_point_file)
|
|
2025
|
+
file_name = basename(entry_point_file)
|
|
2026
|
+
manifest_path = join(directory, "manifest.json")
|
|
2027
|
+
|
|
2028
|
+
if app_mode is None:
|
|
2029
|
+
_, extension = splitext(file_name)
|
|
2030
|
+
app_mode = AppModes.get_by_extension(extension, True)
|
|
2031
|
+
if app_mode == AppModes.UNKNOWN:
|
|
2032
|
+
raise RSConnectException('Could not determine the app mode from "%s"; please specify one.' % extension)
|
|
2033
|
+
|
|
2034
|
+
manifest_data = make_source_manifest(
|
|
2035
|
+
app_mode,
|
|
2036
|
+
environment,
|
|
2037
|
+
file_name,
|
|
2038
|
+
None,
|
|
2039
|
+
image,
|
|
2040
|
+
env_management_py,
|
|
2041
|
+
env_management_r,
|
|
2042
|
+
r_environment,
|
|
2043
|
+
)
|
|
2044
|
+
if hide_all_input or hide_tagged_input:
|
|
2045
|
+
if "jupyter" not in manifest_data:
|
|
2046
|
+
manifest_data["jupyter"] = {}
|
|
2047
|
+
if hide_all_input:
|
|
2048
|
+
manifest_data["jupyter"]["hide_all_input"] = True
|
|
2049
|
+
if hide_tagged_input:
|
|
2050
|
+
manifest_data["jupyter"]["hide_tagged_input"] = True
|
|
2051
|
+
|
|
2052
|
+
manifest_add_file(manifest_data, file_name, directory)
|
|
2053
|
+
manifest_add_buffer(manifest_data, environment.filename, environment.contents)
|
|
2054
|
+
|
|
2055
|
+
for rel_path in extra_files:
|
|
2056
|
+
manifest_add_file(manifest_data, rel_path, directory)
|
|
2057
|
+
|
|
2058
|
+
write_manifest_json(manifest_path, manifest_data)
|
|
2059
|
+
|
|
2060
|
+
return exists(join(directory, environment.filename))
|
|
2061
|
+
|
|
2062
|
+
|
|
2063
|
+
MULTI_NOTEBOOK_EXC_MSG = """
|
|
2064
|
+
Unable to infer entrypoint.
|
|
2065
|
+
Multi-notebook deployments need to be specified with the following:
|
|
2066
|
+
1) A directory as the path
|
|
2067
|
+
2) Set multi_notebook=True,
|
|
2068
|
+
i.e. include --multi-notebook (or -m) in the CLI command.
|
|
2069
|
+
"""
|
|
2070
|
+
|
|
2071
|
+
|
|
2072
|
+
def create_voila_manifest(
|
|
2073
|
+
path: str,
|
|
2074
|
+
entrypoint: Optional[str],
|
|
2075
|
+
environment: Environment,
|
|
2076
|
+
extra_files: Sequence[str],
|
|
2077
|
+
excludes: Sequence[str],
|
|
2078
|
+
force_generate: bool = True,
|
|
2079
|
+
image: Optional[str] = None,
|
|
2080
|
+
env_management_py: Optional[bool] = None,
|
|
2081
|
+
env_management_r: Optional[bool] = None,
|
|
2082
|
+
r_environment: Optional[REnvironment] = None,
|
|
2083
|
+
multi_notebook: bool = False,
|
|
2084
|
+
) -> Manifest:
|
|
2085
|
+
"""
|
|
2086
|
+
Creates and writes a manifest.json file for the given path.
|
|
2087
|
+
|
|
2088
|
+
:param path: the file, or the directory containing the files to deploy.
|
|
2089
|
+
:param entry_point: the main entry point for the API.
|
|
2090
|
+
:param environment: the Python environment to start with. This should be what's
|
|
2091
|
+
returned by the inspect_environment() function.
|
|
2092
|
+
:param app_mode: the application mode to assume. If this is None, the extension
|
|
2093
|
+
portion of the entry point file name will be used to derive one. Previous default = None.
|
|
2094
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
2095
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
2096
|
+
:param force_generate: bool indicating whether to force generate manifest and related environment files.
|
|
2097
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2098
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
2099
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2100
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
2101
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2102
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
2103
|
+
:return: the manifest data structure.
|
|
2104
|
+
"""
|
|
2105
|
+
if not path:
|
|
2106
|
+
raise RSConnectException("A valid path must be provided.")
|
|
2107
|
+
extra_files = list(extra_files) if extra_files else []
|
|
2108
|
+
entrypoint_candidates = infer_entrypoint_candidates(path=abspath(path), mimetype="text/ipynb")
|
|
2109
|
+
|
|
2110
|
+
deploy_dir = guess_deploy_dir(path, entrypoint)
|
|
2111
|
+
if not multi_notebook:
|
|
2112
|
+
if len(entrypoint_candidates) <= 0:
|
|
2113
|
+
if entrypoint is None:
|
|
2114
|
+
raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG)
|
|
2115
|
+
entrypoint = abs_entrypoint(path, entrypoint)
|
|
2116
|
+
elif len(entrypoint_candidates) == 1:
|
|
2117
|
+
if entrypoint:
|
|
2118
|
+
entrypoint = abs_entrypoint(path, entrypoint)
|
|
2119
|
+
else:
|
|
2120
|
+
entrypoint = entrypoint_candidates[0]
|
|
2121
|
+
else: # len(entrypoint_candidates) > 1:
|
|
2122
|
+
if entrypoint is None:
|
|
2123
|
+
raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG)
|
|
2124
|
+
entrypoint = abs_entrypoint(path, entrypoint)
|
|
2125
|
+
|
|
2126
|
+
if multi_notebook:
|
|
2127
|
+
if path and not isdir(path):
|
|
2128
|
+
raise RSConnectException(MULTI_NOTEBOOK_EXC_MSG)
|
|
2129
|
+
_warn_on_ignored_entrypoint(entrypoint)
|
|
2130
|
+
deploy_dir = entrypoint = abspath(path)
|
|
2131
|
+
extra_files = validate_extra_files(deploy_dir, extra_files, use_abspath=True)
|
|
2132
|
+
excludes = list(excludes) if excludes else []
|
|
2133
|
+
excludes.extend([environment.filename, "manifest.json"])
|
|
2134
|
+
excludes.extend(list_environment_dirs(deploy_dir))
|
|
2135
|
+
|
|
2136
|
+
voila_json_path = join(deploy_dir, "voila.json")
|
|
2137
|
+
if isfile(voila_json_path):
|
|
2138
|
+
extra_files.append(voila_json_path)
|
|
2139
|
+
|
|
2140
|
+
manifest = Manifest(
|
|
2141
|
+
app_mode=AppModes.JUPYTER_VOILA,
|
|
2142
|
+
environment=environment,
|
|
2143
|
+
r_environment=r_environment,
|
|
2144
|
+
entrypoint=entrypoint,
|
|
2145
|
+
image=image,
|
|
2146
|
+
env_management_py=env_management_py,
|
|
2147
|
+
env_management_r=env_management_r,
|
|
2148
|
+
)
|
|
2149
|
+
manifest.deploy_dir = deploy_dir
|
|
2150
|
+
if entrypoint and isfile(entrypoint):
|
|
2151
|
+
validate_file_is_notebook(entrypoint)
|
|
2152
|
+
manifest.entrypoint = entrypoint
|
|
2153
|
+
|
|
2154
|
+
manifest.add_to_buffer(join(deploy_dir, environment.filename), environment.contents)
|
|
2155
|
+
|
|
2156
|
+
file_list = create_file_list(path, extra_files, excludes, use_abspath=True)
|
|
2157
|
+
for abs_path in file_list:
|
|
2158
|
+
manifest.add_file(abs_path)
|
|
2159
|
+
return manifest
|
|
2160
|
+
|
|
2161
|
+
|
|
2162
|
+
def write_voila_manifest_json(
|
|
2163
|
+
path: str,
|
|
2164
|
+
entrypoint: Optional[str],
|
|
2165
|
+
environment: Environment,
|
|
2166
|
+
extra_files: Sequence[str],
|
|
2167
|
+
excludes: Sequence[str],
|
|
2168
|
+
force_generate: bool = True,
|
|
2169
|
+
image: Optional[str] = None,
|
|
2170
|
+
env_management_py: Optional[bool] = None,
|
|
2171
|
+
env_management_r: Optional[bool] = None,
|
|
2172
|
+
r_environment: Optional[REnvironment] = None,
|
|
2173
|
+
multi_notebook: bool = False,
|
|
2174
|
+
) -> bool:
|
|
2175
|
+
"""
|
|
2176
|
+
Creates and writes a manifest.json file for the given path.
|
|
2177
|
+
|
|
2178
|
+
:param path: the file, or the directory containing the files to deploy.
|
|
2179
|
+
:param entry_point: the main entry point for the API.
|
|
2180
|
+
:param environment: the Python environment to start with. This should be what's
|
|
2181
|
+
returned by the inspect_environment() function.
|
|
2182
|
+
:param app_mode: the application mode to assume. If this is None, the extension
|
|
2183
|
+
portion of the entry point file name will be used to derive one. Previous default = None.
|
|
2184
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
2185
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
2186
|
+
:param force_generate: bool indicating whether to force generate manifest and related environment files.
|
|
2187
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2188
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
2189
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2190
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
2191
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2192
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
2193
|
+
:return: whether the manifest was written.
|
|
2194
|
+
"""
|
|
2195
|
+
manifest = create_voila_manifest(
|
|
2196
|
+
path=path,
|
|
2197
|
+
entrypoint=entrypoint,
|
|
2198
|
+
environment=environment,
|
|
2199
|
+
extra_files=extra_files,
|
|
2200
|
+
excludes=excludes,
|
|
2201
|
+
force_generate=force_generate,
|
|
2202
|
+
image=image,
|
|
2203
|
+
env_management_py=env_management_py,
|
|
2204
|
+
env_management_r=env_management_r,
|
|
2205
|
+
r_environment=r_environment,
|
|
2206
|
+
multi_notebook=multi_notebook,
|
|
2207
|
+
)
|
|
2208
|
+
|
|
2209
|
+
if manifest.entrypoint is None:
|
|
2210
|
+
raise RSConnectException("Voila manifest requires an entrypoint.")
|
|
2211
|
+
|
|
2212
|
+
deploy_dir = dirname(manifest.entrypoint) if isfile(manifest.entrypoint) else manifest.entrypoint
|
|
2213
|
+
manifest_flattened_copy_data = manifest.get_flattened_copy().data
|
|
2214
|
+
if multi_notebook and "metadata" in manifest_flattened_copy_data:
|
|
2215
|
+
manifest_flattened_copy_data["metadata"]["entrypoint"] = ""
|
|
2216
|
+
manifest_path = join(deploy_dir, "manifest.json")
|
|
2217
|
+
write_manifest_json(manifest_path, manifest_flattened_copy_data)
|
|
2218
|
+
return exists(manifest_path)
|
|
2219
|
+
|
|
2220
|
+
|
|
2221
|
+
def create_api_manifest_and_environment_file(
|
|
2222
|
+
directory: str,
|
|
2223
|
+
entry_point: str,
|
|
2224
|
+
environment: Environment,
|
|
2225
|
+
app_mode: AppMode,
|
|
2226
|
+
extra_files: Sequence[str],
|
|
2227
|
+
excludes: Sequence[str],
|
|
2228
|
+
force: bool,
|
|
2229
|
+
image: Optional[str] = None,
|
|
2230
|
+
env_management_py: Optional[bool] = None,
|
|
2231
|
+
env_management_r: Optional[bool] = None,
|
|
2232
|
+
) -> None:
|
|
2233
|
+
"""
|
|
2234
|
+
Creates and writes a manifest.json file for the given Python API entry point. If
|
|
2235
|
+
the related environment file (requirements.txt, environment.yml, etc.) doesn't
|
|
2236
|
+
exist (or force is set to True), the environment file will also be written.
|
|
2237
|
+
|
|
2238
|
+
:param directory: the root directory of the Python API.
|
|
2239
|
+
:param entry_point: the module/executable object for the WSGi framework.
|
|
2240
|
+
:param environment: the Python environment to start with. This should be what's
|
|
2241
|
+
returned by the inspect_environment() function.
|
|
2242
|
+
:param app_mode: the application mode to assume. Previous default = AppModes.PYTHON_API.
|
|
2243
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
2244
|
+
:param excludes: a sequence of glob patterns that will exclude matched files. Previous default = None.
|
|
2245
|
+
:param force: if True, forces the environment file to be written. even if it
|
|
2246
|
+
already exists. Previous default = True.
|
|
2247
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2248
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
2249
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2250
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
2251
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2252
|
+
:return:
|
|
2253
|
+
"""
|
|
2254
|
+
if (
|
|
2255
|
+
not write_api_manifest_json(
|
|
2256
|
+
directory,
|
|
2257
|
+
entry_point,
|
|
2258
|
+
environment,
|
|
2259
|
+
app_mode,
|
|
2260
|
+
extra_files,
|
|
2261
|
+
excludes,
|
|
2262
|
+
image,
|
|
2263
|
+
env_management_py,
|
|
2264
|
+
env_management_r,
|
|
2265
|
+
)
|
|
2266
|
+
or force
|
|
2267
|
+
):
|
|
2268
|
+
write_environment_file(environment, directory)
|
|
2269
|
+
|
|
2270
|
+
|
|
2271
|
+
def write_api_manifest_json(
|
|
2272
|
+
directory: str,
|
|
2273
|
+
entry_point: str,
|
|
2274
|
+
environment: Environment,
|
|
2275
|
+
app_mode: AppMode,
|
|
2276
|
+
extra_files: Sequence[str],
|
|
2277
|
+
excludes: Sequence[str],
|
|
2278
|
+
image: Optional[str] = None,
|
|
2279
|
+
env_management_py: Optional[bool] = None,
|
|
2280
|
+
env_management_r: Optional[bool] = None,
|
|
2281
|
+
r_environment: Optional[REnvironment] = None,
|
|
2282
|
+
) -> bool:
|
|
2283
|
+
"""
|
|
2284
|
+
Creates and writes a manifest.json file for the given entry point file. If
|
|
2285
|
+
the application mode is not provided, an attempt will be made to resolve one
|
|
2286
|
+
based on the extension portion of the entry point file.
|
|
2287
|
+
|
|
2288
|
+
:param directory: the root directory of the Python API.
|
|
2289
|
+
:param entry_point: the module/executable object for the WSGi framework.
|
|
2290
|
+
:param environment: the Python environment to start with. This should be what's
|
|
2291
|
+
returned by the inspect_environment() function.
|
|
2292
|
+
:param app_mode: the application mode to assume. Previous default = AppModes.PYTHON_API.
|
|
2293
|
+
:param extra_files: any extra files that should be included in the manifest. Previous default = None.
|
|
2294
|
+
:param excludes: a sequence of glob patterns that will exclude matched files. Previous default = None.
|
|
2295
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2296
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
2297
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2298
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
2299
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2300
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
2301
|
+
:return: whether or not the environment file (requirements.txt, environment.yml,
|
|
2302
|
+
etc.) that goes along with the manifest exists.
|
|
2303
|
+
"""
|
|
2304
|
+
extra_files = validate_extra_files(directory, extra_files)
|
|
2305
|
+
manifest, _ = make_api_manifest(
|
|
2306
|
+
directory,
|
|
2307
|
+
entry_point,
|
|
2308
|
+
app_mode,
|
|
2309
|
+
environment,
|
|
2310
|
+
extra_files,
|
|
2311
|
+
excludes,
|
|
2312
|
+
image,
|
|
2313
|
+
env_management_py,
|
|
2314
|
+
env_management_r,
|
|
2315
|
+
r_environment,
|
|
2316
|
+
)
|
|
2317
|
+
manifest_path = join(directory, "manifest.json")
|
|
2318
|
+
|
|
2319
|
+
write_manifest_json(manifest_path, manifest)
|
|
2320
|
+
|
|
2321
|
+
return exists(join(directory, environment.filename))
|
|
2322
|
+
|
|
2323
|
+
|
|
2324
|
+
def write_nodejs_manifest_json(
|
|
2325
|
+
directory: str,
|
|
2326
|
+
entry_point: str,
|
|
2327
|
+
node_environment: NodeEnvironment,
|
|
2328
|
+
extra_files: Sequence[str],
|
|
2329
|
+
excludes: Sequence[str],
|
|
2330
|
+
image: Optional[str] = None,
|
|
2331
|
+
env_management_node: Optional[bool] = None,
|
|
2332
|
+
) -> None:
|
|
2333
|
+
"""
|
|
2334
|
+
Creates and writes a manifest.json file for a Node.js application.
|
|
2335
|
+
|
|
2336
|
+
:param directory: the root directory of the Node.js application.
|
|
2337
|
+
:param entry_point: the entry point file (e.g., "app.js").
|
|
2338
|
+
:param node_environment: the Node.js environment information.
|
|
2339
|
+
:param extra_files: any extra files that should be included in the manifest.
|
|
2340
|
+
:param excludes: a sequence of glob patterns that will exclude matched files.
|
|
2341
|
+
:param image: the optional docker image for off-host execution.
|
|
2342
|
+
:param env_management_node: False prevents Connect from managing the Node.js environment.
|
|
2343
|
+
"""
|
|
2344
|
+
extra_files = validate_extra_files(directory, extra_files)
|
|
2345
|
+
manifest, _ = make_nodejs_manifest(
|
|
2346
|
+
directory,
|
|
2347
|
+
entry_point,
|
|
2348
|
+
node_environment,
|
|
2349
|
+
extra_files,
|
|
2350
|
+
excludes,
|
|
2351
|
+
image,
|
|
2352
|
+
env_management_node,
|
|
2353
|
+
)
|
|
2354
|
+
manifest_path = join(directory, "manifest.json")
|
|
2355
|
+
|
|
2356
|
+
write_manifest_json(manifest_path, manifest)
|
|
2357
|
+
|
|
2358
|
+
|
|
2359
|
+
def write_environment_file(
|
|
2360
|
+
environment: Environment,
|
|
2361
|
+
directory: str,
|
|
2362
|
+
) -> None:
|
|
2363
|
+
"""
|
|
2364
|
+
Writes the environment file (requirements.txt, environment.yml, etc.) to the
|
|
2365
|
+
specified directory.
|
|
2366
|
+
|
|
2367
|
+
:param environment: the Python environment to start with. This should be what's
|
|
2368
|
+
returned by the inspect_environment() function.
|
|
2369
|
+
:param directory: the directory where the file should be written.
|
|
2370
|
+
"""
|
|
2371
|
+
environment_file_path = join(directory, environment.filename)
|
|
2372
|
+
with open(environment_file_path, "w") as f:
|
|
2373
|
+
f.write(environment.contents)
|
|
2374
|
+
|
|
2375
|
+
|
|
2376
|
+
def describe_manifest(
|
|
2377
|
+
file_name: str,
|
|
2378
|
+
) -> tuple[str | None, str | None]:
|
|
2379
|
+
"""
|
|
2380
|
+
Determine the entry point and/or primary file from the given manifest file.
|
|
2381
|
+
If no entry point is recorded in the manifest, then None will be returned for
|
|
2382
|
+
that. The same is true for the primary document. None will be returned for
|
|
2383
|
+
both if the file doesn't exist or doesn't look like a manifest file.
|
|
2384
|
+
|
|
2385
|
+
:param file_name: the name of the manifest file to read.
|
|
2386
|
+
:return: the entry point and primary document from the manifest.
|
|
2387
|
+
"""
|
|
2388
|
+
if basename(file_name) == "manifest.json" and exists(file_name):
|
|
2389
|
+
manifest, _ = read_manifest_file(file_name)
|
|
2390
|
+
metadata = manifest.get("metadata")
|
|
2391
|
+
if metadata:
|
|
2392
|
+
# noinspection SpellCheckingInspection
|
|
2393
|
+
return (
|
|
2394
|
+
metadata.get("entrypoint"),
|
|
2395
|
+
metadata.get("primary_rmd") or metadata.get("primary_html"),
|
|
2396
|
+
)
|
|
2397
|
+
return None, None
|
|
2398
|
+
|
|
2399
|
+
|
|
2400
|
+
def write_quarto_manifest_json(
|
|
2401
|
+
file_or_directory: str,
|
|
2402
|
+
inspect: QuartoInspectResult,
|
|
2403
|
+
app_mode: AppMode,
|
|
2404
|
+
environment: Optional[Environment],
|
|
2405
|
+
extra_files: Sequence[str],
|
|
2406
|
+
excludes: Sequence[str],
|
|
2407
|
+
image: Optional[str] = None,
|
|
2408
|
+
env_management_py: Optional[bool] = None,
|
|
2409
|
+
env_management_r: Optional[bool] = None,
|
|
2410
|
+
r_environment: Optional[REnvironment] = None,
|
|
2411
|
+
) -> None:
|
|
2412
|
+
"""
|
|
2413
|
+
Creates and writes a manifest.json file for the given Quarto project.
|
|
2414
|
+
|
|
2415
|
+
:param file_or_directory: The Quarto document or the directory containing the Quarto project.
|
|
2416
|
+
:param inspect: The parsed JSON from a 'quarto inspect' against the project.
|
|
2417
|
+
:param app_mode: The application mode to assume (such as AppModes.STATIC_QUARTO)
|
|
2418
|
+
:param environment: The (optional) Python environment to use.
|
|
2419
|
+
:param extra_files: Any extra files to include in the manifest.
|
|
2420
|
+
:param excludes: A sequence of glob patterns to exclude when enumerating files to bundle.
|
|
2421
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2422
|
+
:param env_management_py: False prevents Connect from managing the Python environment for this bundle.
|
|
2423
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2424
|
+
:param env_management_r: False prevents Connect from managing the R environment for this bundle.
|
|
2425
|
+
The server administrator is responsible for installing packages in the runtime environment. Default = None.
|
|
2426
|
+
:param r_environment: optional R dependencies detected from renv.lock to add to the manifest.
|
|
2427
|
+
"""
|
|
2428
|
+
|
|
2429
|
+
manifest, _ = make_quarto_manifest(
|
|
2430
|
+
file_or_directory,
|
|
2431
|
+
inspect,
|
|
2432
|
+
app_mode,
|
|
2433
|
+
environment,
|
|
2434
|
+
extra_files,
|
|
2435
|
+
excludes,
|
|
2436
|
+
image,
|
|
2437
|
+
env_management_py,
|
|
2438
|
+
env_management_r,
|
|
2439
|
+
r_environment,
|
|
2440
|
+
)
|
|
2441
|
+
|
|
2442
|
+
base_dir = file_or_directory
|
|
2443
|
+
if not isdir(file_or_directory):
|
|
2444
|
+
base_dir = dirname(file_or_directory)
|
|
2445
|
+
manifest_path = join(base_dir, "manifest.json")
|
|
2446
|
+
write_manifest_json(manifest_path, manifest)
|
|
2447
|
+
|
|
2448
|
+
|
|
2449
|
+
def write_tensorflow_manifest_json(
|
|
2450
|
+
directory: str,
|
|
2451
|
+
extra_files: Sequence[str],
|
|
2452
|
+
excludes: Sequence[str],
|
|
2453
|
+
image: Optional[str] = None,
|
|
2454
|
+
) -> None:
|
|
2455
|
+
"""
|
|
2456
|
+
Creates and writes a manifest.json file for the given TensorFlow content.
|
|
2457
|
+
|
|
2458
|
+
:param directory: The directory containing the TensorFlow model.
|
|
2459
|
+
:param environment: The (optional) Python environment to use.
|
|
2460
|
+
:param extra_files: Any extra files to include in the manifest.
|
|
2461
|
+
:param excludes: A sequence of glob patterns to exclude when enumerating files to bundle.
|
|
2462
|
+
:param image: the optional docker image to be specified for off-host execution. Default = None.
|
|
2463
|
+
"""
|
|
2464
|
+
|
|
2465
|
+
manifest = make_tensorflow_manifest(
|
|
2466
|
+
directory,
|
|
2467
|
+
extra_files,
|
|
2468
|
+
excludes,
|
|
2469
|
+
image,
|
|
2470
|
+
)
|
|
2471
|
+
manifest_path = join(directory, "manifest.json")
|
|
2472
|
+
write_manifest_json(manifest_path, manifest)
|
|
2473
|
+
|
|
2474
|
+
|
|
2475
|
+
def write_manifest_json(manifest_path: str | Path, manifest: ManifestData) -> None:
|
|
2476
|
+
"""
|
|
2477
|
+
Write the manifest data as JSON to the named manifest.json with a trailing newline.
|
|
2478
|
+
"""
|
|
2479
|
+
with open(manifest_path, "w") as f:
|
|
2480
|
+
json.dump(manifest, f, indent=2)
|
|
2481
|
+
f.write("\n")
|