shipit-cli 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
shipit/cli.py CHANGED
@@ -1,47 +1,40 @@
1
1
  import tempfile
2
- import hashlib
3
- import requests
4
2
  import os
5
- import shlex
6
- import shutil
7
3
  import sys
8
4
  import json
9
- import yaml
10
- import base64
11
- from dataclasses import dataclass
12
5
  from pathlib import Path
13
- from typing import (
14
- Any,
15
- Dict,
16
- List,
17
- Optional,
18
- Protocol,
19
- Set,
20
- Tuple,
21
- TypedDict,
22
- Union,
23
- Literal,
24
- cast,
25
- )
26
- from shutil import copy, copytree, ignore_patterns
6
+ from dataclasses import dataclass
7
+ from typing import Any, Dict, List, Optional, Tuple, cast, Literal
27
8
 
28
- import sh # type: ignore[import-untyped]
29
- import starlark as sl
9
+ import xingque as sl
30
10
  import typer
31
11
  from rich import box
32
- from rich.console import Console
33
12
  from rich.panel import Panel
34
- from rich.rule import Rule
35
13
  from rich.syntax import Syntax
36
14
 
37
- from shipit.version import version as shipit_version
38
- from shipit.generator import generate_shipit, detect_provider
39
- from shipit.providers.base import CustomCommands
40
- from shipit.procfile import Procfile
15
+ from shipit.generator import generate_shipit, load_provider, load_provider_config
16
+ from shipit.providers.base import Config
41
17
  from dotenv import dotenv_values
42
-
43
-
44
- console = Console()
18
+ from shipit.builders import BuildBackend, DockerBuildBackend, LocalBuildBackend
19
+ from shipit.runners import Runner, LocalRunner, WasmerRunner
20
+ from shipit.shipit_types import (
21
+ Build,
22
+ CopyStep,
23
+ EnvStep,
24
+ Mount,
25
+ Package,
26
+ PathStep,
27
+ PrepareStep,
28
+ RunStep,
29
+ Serve,
30
+ Service,
31
+ Step,
32
+ UseStep,
33
+ Volume,
34
+ WorkdirStep,
35
+ )
36
+ from shipit.ui import console
37
+ from shipit.version import version as shipit_version
45
38
 
46
39
  app = typer.Typer(invoke_without_command=True)
47
40
 
@@ -50,1157 +43,16 @@ ASSETS_PATH = DIR_PATH / "assets"
50
43
 
51
44
 
52
45
  @dataclass
53
- class Mount:
54
- name: str
55
- build_path: Path
56
- serve_path: Path
57
-
58
-
59
- @dataclass
60
- class Volume:
61
- name: str
62
- serve_path: Path
63
-
64
-
65
- @dataclass
66
- class Service:
67
- name: str
68
- provider: Literal[
69
- "postgres", "mysql", "redis"
70
- ] # Right now we only support postgres and mysql
71
-
72
-
73
- @dataclass
74
- class Serve:
75
- name: str
76
- provider: str
77
- build: List["Step"]
78
- deps: List["Package"]
79
- commands: Dict[str, str]
80
- cwd: Optional[str] = None
81
- prepare: Optional[List["PrepareStep"]] = None
82
- workers: Optional[List[str]] = None
83
- mounts: Optional[List[Mount]] = None
84
- volumes: Optional[List[Volume]] = None
85
- env: Optional[Dict[str, str]] = None
86
- services: Optional[List[Service]] = None
87
-
88
-
89
- @dataclass
90
- class Package:
91
- name: str
92
- version: Optional[str] = None
93
- architecture: Optional[Literal["64-bit", "32-bit"]] = None
94
-
95
- def __str__(self) -> str: # pragma: no cover - simple representation
96
- name = f"{self.name}({self.architecture})" if self.architecture else self.name
97
- if self.version is None:
98
- return name
99
- return f"{name}@{self.version}"
100
-
101
-
102
- @dataclass
103
- class RunStep:
104
- command: str
105
- inputs: Optional[List[str]] = None
106
- outputs: Optional[List[str]] = None
107
- group: Optional[str] = None
108
-
109
-
110
- @dataclass
111
- class WorkdirStep:
112
- path: Path
113
-
114
-
115
- @dataclass
116
- class CopyStep:
117
- source: str
118
- target: str
119
- ignore: Optional[List[str]] = None
120
- # We can copy from the app source or from the shipit assets folder
121
- base: Literal["source", "assets"] = "source"
122
-
123
- def is_download(self) -> bool:
124
- return self.source.startswith("http://") or self.source.startswith("https://")
125
-
126
-
127
- @dataclass
128
- class EnvStep:
129
- variables: Dict[str, str]
130
-
131
- def __str__(self) -> str: # pragma: no cover - simple representation
132
- return " ".join([f"{key}={value}" for key, value in self.variables.items()])
133
-
134
-
135
- @dataclass
136
- class UseStep:
137
- dependencies: List[Package]
138
-
139
-
140
- @dataclass
141
- class PathStep:
46
+ class CtxMount:
47
+ ref: str
142
48
  path: str
143
-
144
-
145
- Step = Union[RunStep, CopyStep, EnvStep, PathStep, UseStep, WorkdirStep]
146
- PrepareStep = Union[RunStep]
147
-
148
-
149
- @dataclass
150
- class Build:
151
- deps: List[Package]
152
- steps: List[Step]
153
-
154
-
155
- def write_stdout(line: str) -> None:
156
- sys.stdout.write(line) # print to console
157
- sys.stdout.flush()
158
-
159
-
160
- def write_stderr(line: str) -> None:
161
- sys.stderr.write(line) # print to console
162
- sys.stderr.flush()
163
-
164
-
165
- class MapperItem(TypedDict):
166
- dependencies: Dict[str, str]
167
- scripts: Set[str]
168
- env: Dict[str, str]
169
- aliases: Dict[str, str]
170
-
171
-
172
- class Builder(Protocol):
173
- def build(
174
- self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
175
- ) -> None: ...
176
- def build_prepare(self, serve: Serve) -> None: ...
177
- def build_serve(self, serve: Serve) -> None: ...
178
- def finalize_build(self, serve: Serve) -> None: ...
179
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None: ...
180
- def getenv(self, name: str) -> Optional[str]: ...
181
- def run_serve_command(self, command: str) -> None: ...
182
- def run_command(
183
- self, command: str, extra_args: Optional[List[str]] | None = None
184
- ) -> Any: ...
185
- def get_build_mount_path(self, name: str) -> Path: ...
186
- def get_serve_mount_path(self, name: str) -> Path: ...
187
-
188
-
189
- class DockerBuilder:
190
- mise_mapper = {
191
- "php": {
192
- "source": "ubi:adwinying/php",
193
- },
194
- "composer": {
195
- "source": "ubi:composer/composer",
196
- "postinstall": """composer_dir=$(mise where ubi:composer/composer); ln -s "$composer_dir/composer.phar" /usr/local/bin/composer""",
197
- },
198
- }
199
-
200
- def __init__(self, src_dir: Path, docker_client: Optional[str] = None) -> None:
201
- self.src_dir = src_dir
202
- self.docker_file_contents = ""
203
- self.docker_path = self.src_dir / ".shipit" / "docker"
204
- self.docker_out_path = self.docker_path / "out"
205
- self.depot_metadata = self.docker_path / "depot-build.json"
206
- self.docker_file_path = self.docker_path / "Dockerfile"
207
- self.docker_name_path = self.docker_path / "name"
208
- self.docker_ignore_path = self.docker_path / "Dockerfile.dockerignore"
209
- self.shipit_docker_path = Path("/shipit")
210
- self.docker_client = docker_client or "docker"
211
- self.env = {
212
- "HOME": "/root",
213
- }
214
-
215
- def get_mount_path(self, name: str) -> Path:
216
- if name == "app":
217
- return Path("app")
218
- else:
219
- return Path("opt") / name
220
-
221
- def get_build_mount_path(self, name: str) -> Path:
222
- path = Path("/") / self.get_mount_path(name)
223
- return path
224
-
225
- def get_serve_mount_path(self, name: str) -> Path:
226
- return self.docker_out_path / self.get_mount_path(name)
227
-
228
- @property
229
- def is_depot(self) -> bool:
230
- return self.docker_client == "depot"
231
-
232
- def getenv(self, name: str) -> Optional[str]:
233
- return self.env.get(name) or os.environ.get(name)
234
-
235
- def mkdir(self, path: Path) -> Path:
236
- path = self.shipit_docker_path / path
237
- self.docker_file_contents += f"RUN mkdir -p {str(path.absolute())}\n"
238
- return path.absolute()
239
-
240
- def build_dockerfile(self, image_name: str) -> None:
241
- self.docker_file_path.write_text(self.docker_file_contents)
242
- self.docker_name_path.write_text(image_name)
243
- self.print_dockerfile()
244
- extra_args = []
245
- # if self.is_depot:
246
- # # We load the docker image back into the local docker daemon
247
- # # extra_args += ["--load"]
248
- # extra_args += ["--save", f"--metadata-file={self.depot_metadata.absolute()}"]
249
- sh.Command(self.docker_client)(
250
- "build",
251
- "-f",
252
- (self.docker_path / "Dockerfile").absolute(),
253
- "-t",
254
- image_name,
255
- "--platform",
256
- "linux/amd64",
257
- "--output",
258
- self.docker_out_path.absolute(),
259
- ".",
260
- *extra_args,
261
- _cwd=self.src_dir.absolute(),
262
- _env=os.environ, # Pass the current environment variables to the Docker client
263
- _out=write_stdout,
264
- _err=write_stderr,
265
- )
266
- # if self.is_depot:
267
- # json_text = self.depot_metadata.read_text()
268
- # json_data = json.loads(json_text)
269
- # build_data = json_data["depot.build"]
270
- # image_id = build_data["buildID"]
271
- # project = build_data["projectID"]
272
- # sh.Command("depot")(
273
- # "pull",
274
- # "--platform",
275
- # "linux/amd64",
276
- # "--project",
277
- # project,
278
- # image_id,
279
- # _cwd=self.src_dir.absolute(),
280
- # _env=os.environ, # Pass the current environment variables to the Docker client
281
- # _out=write_stdout,
282
- # _err=write_stderr,
283
- # )
284
- # # console.print(f"[bold]Image ID:[/bold] {image_id}")
285
-
286
- def finalize_build(self, serve: Serve) -> None:
287
- console.print(f"\n[bold]Building Docker file[/bold]")
288
- self.build_dockerfile(serve.name)
289
- console.print(Rule(characters="-", style="bright_black"))
290
- console.print(f"[bold]Build complete ✅[/bold]")
291
-
292
- def run_command(self, command: str, extra_args: Optional[List[str]] = None) -> Any:
293
- image_name = self.docker_name_path.read_text()
294
- docker_args: List[str] = [
295
- "run",
296
- "-p",
297
- "80:80",
298
- "--rm",
299
- ]
300
- # Attach volumes if present
301
- # if serve.volumes:
302
- # for vol in serve.volumes:
303
- # docker_args += [
304
- # "--mount",
305
- # f"type=volume,source={vol.name},target={str(vol.serve_path)}",
306
- # ]
307
- return sh.Command("docker")(
308
- *docker_args,
309
- image_name,
310
- command,
311
- *(extra_args or []),
312
- _env={
313
- "DOCKER_BUILDKIT": "1",
314
- **os.environ,
315
- }, # Pass the current environment variables to the Docker client
316
- _out=write_stdout,
317
- _err=write_stderr,
318
- )
319
-
320
- def create_file(self, path: Path, content: str, mode: int = 0o755) -> Path:
321
- # docker_files = self.docker_path / "files" / path.name
322
- # docker_files.write_text(content)
323
- # docker_files.chmod(mode)
324
- self.docker_file_contents += f"""
325
- RUN cat > {path.absolute()} <<'EOF'
326
- {content}
327
- EOF
328
-
329
- RUN chmod {oct(mode)[2:]} {path.absolute()}
330
- """
331
-
332
- return path.absolute()
333
-
334
- def print_dockerfile(self) -> None:
335
- docker_file = self.docker_path / "Dockerfile"
336
- manifest_panel = Panel(
337
- Syntax(
338
- docker_file.read_text(),
339
- "dockerfile",
340
- theme="monokai",
341
- background_color="default",
342
- line_numbers=True,
343
- ),
344
- box=box.SQUARE,
345
- border_style="bright_black",
346
- expand=False,
347
- )
348
- console.print(manifest_panel, markup=False, highlight=True)
349
-
350
- def add_dependency(self, dependency: Package):
351
- if dependency.name == "pie":
352
- self.docker_file_contents += f"RUN apt-get update && apt-get -y --no-install-recommends install gcc make autoconf libtool bison re2c pkg-config libpq-dev\n"
353
- self.docker_file_contents += f"RUN curl -L --output /usr/bin/pie https://github.com/php/pie/releases/download/1.2.0/pie.phar && chmod +x /usr/bin/pie\n"
354
- return
355
- elif dependency.name == "static-web-server":
356
- if dependency.version:
357
- self.docker_file_contents += (
358
- f"ENV SWS_INSTALL_VERSION={dependency.version}\n"
359
- )
360
- self.docker_file_contents += f"RUN curl --proto '=https' --tlsv1.2 -sSfL https://get.static-web-server.net | sh\n"
361
- return
362
-
363
- mapped_dependency = self.mise_mapper.get(dependency.name, {})
364
- package_name = mapped_dependency.get("source", dependency.name)
365
- if dependency.version:
366
- self.docker_file_contents += (
367
- f"RUN mise use --global {package_name}@{dependency.version}\n"
368
- )
369
- else:
370
- self.docker_file_contents += f"RUN mise use --global {package_name}\n"
371
- if mapped_dependency.get("postinstall"):
372
- self.docker_file_contents += f"RUN {mapped_dependency.get('postinstall')}\n"
373
-
374
- def build(
375
- self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
376
- ) -> None:
377
- base_path = self.docker_path
378
- shutil.rmtree(base_path, ignore_errors=True)
379
- base_path.mkdir(parents=True, exist_ok=True)
380
- self.docker_file_contents = "# syntax=docker/dockerfile:1.7-labs\n"
381
- self.docker_file_contents += "FROM debian:trixie-slim AS build\n"
382
-
383
- self.docker_file_contents += """
384
- RUN apt-get update \\
385
- && apt-get -y --no-install-recommends install \\
386
- build-essential gcc make autoconf libtool bison \\
387
- dpkg-dev pkg-config re2c locate \\
388
- libmariadb-dev libmariadb-dev-compat libpq-dev \\
389
- libvips-dev default-libmysqlclient-dev libmagickwand-dev \\
390
- libicu-dev libxml2-dev libxslt-dev libyaml-dev \\
391
- sudo curl ca-certificates \\
392
- && rm -rf /var/lib/apt/lists/*
393
-
394
- SHELL ["/bin/bash", "-o", "pipefail", "-c"]
395
- ENV MISE_DATA_DIR="/mise"
396
- ENV MISE_CONFIG_DIR="/mise"
397
- ENV MISE_CACHE_DIR="/mise/cache"
398
- ENV MISE_INSTALL_PATH="/usr/local/bin/mise"
399
- ENV PATH="/mise/shims:$PATH"
400
- # ENV MISE_VERSION="..."
401
-
402
- RUN curl https://mise.run | sh
403
- """
404
- # docker_file_contents += "RUN curl https://mise.run | sh\n"
405
- # self.docker_file_contents += """
406
- # RUN curl https://get.wasmer.io -sSfL | sh -s "v6.1.0-rc.3"
407
- # ENV PATH="/root/.wasmer/bin:${PATH}"
408
- # """
409
- for mount in mounts:
410
- self.docker_file_contents += f"RUN mkdir -p {mount.build_path.absolute()}\n"
411
-
412
- for step in steps:
413
- if isinstance(step, WorkdirStep):
414
- self.docker_file_contents += f"WORKDIR {step.path.absolute()}\n"
415
- elif isinstance(step, RunStep):
416
- if step.inputs:
417
- pre = "\\\n " + "".join(
418
- [
419
- f"--mount=type=bind,source={input},target={input} \\\n "
420
- for input in step.inputs
421
- ]
422
- )
423
- else:
424
- pre = ""
425
- self.docker_file_contents += f"RUN {pre}{step.command}\n"
426
- elif isinstance(step, CopyStep):
427
- if step.is_download():
428
- self.docker_file_contents += (
429
- "ADD " + step.source + " " + step.target + "\n"
430
- )
431
- elif step.base == "assets":
432
- # Detect if the asset exists and is a file
433
- if (ASSETS_PATH / step.source).is_file():
434
- # Read the file content and write it to the target file
435
- content_base64 = base64.b64encode(
436
- (ASSETS_PATH / step.source).read_bytes()
437
- ).decode("utf-8")
438
- self.docker_file_contents += (
439
- f"RUN echo '{content_base64}' | base64 -d > {step.target}\n"
440
- )
441
- elif (ASSETS_PATH / step.source).is_dir():
442
- raise Exception(
443
- f"Asset {step.source} is a directory, shipit doesn't currently support coppying assets directories inside Docker"
444
- )
445
- else:
446
- raise Exception(f"Asset {step.source} does not exist")
447
- else:
448
- if step.ignore:
449
- exclude = (
450
- " \\\n"
451
- + " \\\n".join(
452
- [f" --exclude={ignore}" for ignore in step.ignore]
453
- )
454
- + " \\\n "
455
- )
456
- else:
457
- exclude = ""
458
- self.docker_file_contents += (
459
- f"COPY{exclude} {step.source} {step.target}\n"
460
- )
461
- elif isinstance(step, EnvStep):
462
- env_vars = " ".join(
463
- [f"{key}={value}" for key, value in step.variables.items()]
464
- )
465
- self.docker_file_contents += f"ENV {env_vars}\n"
466
- elif isinstance(step, PathStep):
467
- self.docker_file_contents += f"ENV PATH={step.path}:$PATH\n"
468
- elif isinstance(step, UseStep):
469
- for dependency in step.dependencies:
470
- self.add_dependency(dependency)
471
-
472
- self.docker_file_contents += """
473
- FROM scratch
474
- """
475
- for mount in mounts:
476
- self.docker_file_contents += (
477
- f"COPY --from=build {mount.build_path} {mount.build_path}\n"
478
- )
479
-
480
- self.docker_ignore_path.write_text("""
481
- .shipit
482
- Shipit
483
- """)
484
-
485
- def get_path(self) -> Path:
486
- return Path("/")
487
-
488
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
489
- raise NotImplementedError
490
-
491
- def build_serve(self, serve: Serve) -> None:
492
- serve_command_path = self.mkdir(Path("serve") / "bin")
493
- console.print(f"[bold]Serve Commands:[/bold]")
494
- for dep in serve.deps:
495
- self.add_dependency(dep)
496
-
497
- for command in serve.commands:
498
- console.print(f"* {command}")
499
- command_path = serve_command_path / command
500
- self.create_file(
501
- command_path,
502
- f"#!/bin/bash\ncd {serve.cwd}\n{serve.commands[command]}",
503
- mode=0o755,
504
- )
505
-
506
- def run_serve_command(self, command: str) -> None:
507
- path = self.shipit_docker_path / "serve" / "bin" / command
508
- self.run_command(str(path))
509
-
510
-
511
- class LocalBuilder:
512
- def __init__(self, src_dir: Path) -> None:
513
- self.src_dir = src_dir
514
- self.local_path = self.src_dir / ".shipit" / "local"
515
- self.serve_bin_path = self.local_path / "serve" / "bin"
516
- self.prepare_bash_script = self.local_path / "prepare" / "prepare.sh"
517
- self.build_path = self.local_path / "build"
518
- self.workdir = self.build_path
519
-
520
- def get_mount_path(self, name: str) -> Path:
521
- if name == "app":
522
- return self.build_path / "app"
523
- else:
524
- return self.build_path / "opt" / name
525
-
526
- def get_build_mount_path(self, name: str) -> Path:
527
- return self.get_mount_path(name)
528
-
529
- def get_serve_mount_path(self, name: str) -> Path:
530
- return self.get_mount_path(name)
531
-
532
- def execute_step(self, step: Step, env: Dict[str, str]) -> None:
533
- build_path = self.workdir
534
- if isinstance(step, UseStep):
535
- console.print(
536
- f"[bold]Using dependencies:[/bold] {', '.join([str(dep) for dep in step.dependencies])}"
537
- )
538
- elif isinstance(step, WorkdirStep):
539
- console.print(f"[bold]Working in {step.path}[/bold]")
540
- self.workdir = step.path
541
- # We make sure the dir exists
542
- step.path.mkdir(parents=True, exist_ok=True)
543
- elif isinstance(step, RunStep):
544
- extra = ""
545
- if step.inputs:
546
- for input in step.inputs:
547
- print(f"Copying {input} to {build_path / input}")
548
- copy((self.src_dir / input), (build_path / input))
549
- all_inputs = ", ".join(step.inputs)
550
- extra = f" [bright_black]# using {all_inputs}[/bright_black]"
551
- console.print(
552
- f"[bright_black]$[/bright_black] [bold]{step.command}[/bold]{extra}"
553
- )
554
- command_line = step.command
555
- parts = shlex.split(command_line)
556
- program = parts[0]
557
- extended_paths = [
558
- str(build_path / path) for path in env["PATH"].split(os.pathsep)
559
- ]
560
- extended_paths.append(os.environ["PATH"])
561
- PATH = os.pathsep.join(extended_paths) # type: ignore
562
- exe = shutil.which(program, path=PATH)
563
- if not exe:
564
- raise Exception(f"Program is not installed: {program}")
565
- cmd = sh.Command("bash") # "grep"
566
- result = cmd(
567
- "-c",
568
- command_line,
569
- _env={**env, "PATH": PATH},
570
- _cwd=build_path,
571
- _out=write_stdout,
572
- _err=write_stderr,
573
- )
574
- elif isinstance(step, CopyStep):
575
- ignore_extra = ""
576
- if step.ignore:
577
- ignore_extra = (
578
- f" [bright_black]# ignoring {', '.join(step.ignore)}[/bright_black]"
579
- )
580
- ignore_matches = step.ignore if step.ignore else []
581
- ignore_matches.append(".shipit")
582
- ignore_matches.append("Shipit")
583
-
584
- if step.is_download():
585
- console.print(
586
- f"[bold]Download from {step.source} to {step.target}[/bold]"
587
- )
588
- download_file(step.source, (build_path / step.target))
589
- else:
590
- if step.base == "source":
591
- base = self.src_dir
592
- elif step.base == "assets":
593
- base = ASSETS_PATH
594
- else:
595
- raise Exception(f"Unknown base: {step.base}")
596
-
597
- console.print(
598
- f"[bold]Copy to {step.target} from {step.source}[/bold]{ignore_extra}"
599
- )
600
-
601
- if (base / step.source).is_dir():
602
- copytree(
603
- (base / step.source),
604
- (build_path / step.target),
605
- dirs_exist_ok=True,
606
- ignore=ignore_patterns(*ignore_matches),
607
- )
608
- elif (base / step.source).is_file():
609
- copy(
610
- (base / step.source),
611
- (build_path / step.target),
612
- )
613
- else:
614
- raise Exception(f"Source {step.source} is not a file or directory")
615
- elif isinstance(step, EnvStep):
616
- print(f"Setting environment variables: {step}")
617
- env.update(step.variables)
618
- elif isinstance(step, PathStep):
619
- console.print(f"[bold]Add {step.path}[/bold] to PATH")
620
- fullpath = step.path
621
- env["PATH"] = f"{fullpath}{os.pathsep}{env['PATH']}"
622
- else:
623
- raise Exception(f"Unknown step type: {type(step)}")
624
-
625
- def build(
626
- self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
627
- ) -> None:
628
- console.print(f"\n[bold]Building... 🚀[/bold]")
629
- base_path = self.local_path
630
- shutil.rmtree(base_path, ignore_errors=True)
631
- base_path.mkdir(parents=True, exist_ok=True)
632
- self.build_path.mkdir(exist_ok=True)
633
- for mount in mounts:
634
- mount.build_path.mkdir(parents=True, exist_ok=True)
635
- for step in steps:
636
- console.print(Rule(characters="-", style="bright_black"))
637
- self.execute_step(step, env)
638
-
639
- if "PATH" in env:
640
- path = base_path / ".path"
641
- path.write_text(env["PATH"]) # type: ignore
642
-
643
- console.print(Rule(characters="-", style="bright_black"))
644
- console.print(f"[bold]Build complete ✅[/bold]")
645
-
646
- def mkdir(self, path: Path) -> Path:
647
- path = self.get_path() / path
648
- path.mkdir(parents=True, exist_ok=True)
649
- return path.absolute()
650
-
651
- def create_file(self, path: Path, content: str, mode: int = 0o755) -> Path:
652
- path.write_text(content)
653
- path.chmod(mode)
654
- return path.absolute()
655
-
656
- def run_command(self, command: str, extra_args: Optional[List[str]] = None) -> Any:
657
- return sh.Command(command)(
658
- *(extra_args or []),
659
- _out=write_stdout,
660
- _err=write_stderr,
661
- _env=os.environ,
662
- )
663
-
664
- def getenv(self, name: str) -> Optional[str]:
665
- return os.environ.get(name)
666
-
667
- def get_path(self) -> Path:
668
- return self.local_path
669
-
670
- def build_prepare(self, serve: Serve) -> None:
671
- self.prepare_bash_script.parent.mkdir(parents=True, exist_ok=True)
672
- commands: List[str] = []
673
- if serve.cwd:
674
- commands.append(f"cd {serve.cwd}")
675
- if serve.prepare:
676
- for step in serve.prepare:
677
- if isinstance(step, RunStep):
678
- commands.append(step.command)
679
- elif isinstance(step, WorkdirStep):
680
- commands.append(f"cd {step.path}")
681
- content = "#!/bin/bash\n{body}".format(body="\n".join(commands))
682
- console.print(
683
- f"\n[bold]Created prepare.sh script to run before packaging ✅[/bold]"
684
- )
685
- manifest_panel = Panel(
686
- Syntax(
687
- content,
688
- "bash",
689
- theme="monokai",
690
- background_color="default",
691
- line_numbers=True,
692
- ),
693
- box=box.SQUARE,
694
- border_style="bright_black",
695
- expand=False,
696
- )
697
- console.print(manifest_panel, markup=False, highlight=True)
698
- self.prepare_bash_script.write_text(content)
699
- self.prepare_bash_script.chmod(0o755)
700
-
701
- def finalize_build(self, serve: Serve) -> None:
702
- pass
703
-
704
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
705
- sh.Command(f"{self.prepare_bash_script.absolute()}")(
706
- _out=write_stdout, _err=write_stderr
707
- )
708
-
709
- def build_serve(self, serve: Serve) -> None:
710
- # Remember serve configuration for run-time
711
- console.print("\n[bold]Building serve[/bold]")
712
- self.serve_bin_path.mkdir(parents=True, exist_ok=False)
713
- path = self.get_path() / ".path"
714
- path_text = path.read_text()
715
- console.print(f"[bold]Serve Commands:[/bold]")
716
- for command in serve.commands:
717
- console.print(f"* {command}")
718
- command_path = self.serve_bin_path / command
719
- env_vars = ""
720
- if serve.env:
721
- env_vars = " ".join([f"{k}={v}" for k, v in serve.env.items()])
722
-
723
- content = f"#!/bin/bash\ncd {serve.cwd}\nPATH={path_text}:$PATH {env_vars} {serve.commands[command]}"
724
- command_path.write_text(content)
725
- manifest_panel = Panel(
726
- Syntax(
727
- content.strip(),
728
- "bash",
729
- theme="monokai",
730
- background_color="default",
731
- line_numbers=True,
732
- ),
733
- box=box.SQUARE,
734
- border_style="bright_black",
735
- expand=False,
736
- )
737
- console.print(manifest_panel, markup=False, highlight=True)
738
- command_path.chmod(0o755)
739
-
740
- def run_serve_command(self, command: str) -> None:
741
- console.print(f"\n[bold]Running {command} command[/bold]")
742
- command_path = self.serve_bin_path / command
743
- sh.Command(str(command_path))(_out=write_stdout, _err=write_stderr)
744
-
745
-
746
- class WasmerBuilder:
747
- def get_build_mount_path(self, name: str) -> Path:
748
- return self.inner_builder.get_build_mount_path(name)
749
-
750
- def get_serve_mount_path(self, name: str) -> Path:
751
- if name == "app":
752
- return Path("/app")
753
- else:
754
- return Path("/opt") / name
755
-
756
- mapper: Dict[str, MapperItem] = {
757
- "python": {
758
- "dependencies": {
759
- "latest": "python/python@=3.13.1",
760
- "3.13": "python/python@=3.13.1",
761
- },
762
- "scripts": {"python"},
763
- "aliases": {},
764
- "env": {
765
- "PYTHONEXECUTABLE": "/bin/python",
766
- "PYTHONHOME": "/cpython",
767
- },
768
- },
769
- "pandoc": {
770
- "dependencies": {
771
- "latest": "wasmer/pandoc@=0.0.1",
772
- "3.5": "wasmer/pandoc@=0.0.1",
773
- },
774
- "scripts": {"pandoc"},
775
- },
776
- "ffmpeg": {
777
- "dependencies": {
778
- "latest": "wasmer/ffmpeg@=1.0.5",
779
- "N-111519": "wasmer/ffmpeg@=1.0.5",
780
- },
781
- "scripts": {"ffmpeg"},
782
- },
783
- "php": {
784
- "dependencies": {
785
- "latest": "php/php-32@=8.3.2102",
786
- "8.3": "php/php-32@=8.3.2102",
787
- "8.2": "php/php-32@=8.2.2801",
788
- "8.1": "php/php-32@=8.1.3201",
789
- "7.4": "php/php-32@=7.4.3301",
790
- },
791
- "architecture_dependencies": {
792
- "64-bit": {
793
- "latest": "php/php-64@=8.3.2102",
794
- "8.3": "php/php-64@=8.3.2102",
795
- "8.2": "php/php-64@=8.2.2801",
796
- "8.1": "php/php-64@=8.1.3201",
797
- "7.4": "php/php-64@=7.4.3301",
798
- },
799
- "32-bit": {
800
- "latest": "php/php-32@=8.3.2102",
801
- "8.3": "php/php-32@=8.3.2102",
802
- "8.2": "php/php-32@=8.2.2801",
803
- "8.1": "php/php-32@=8.1.3201",
804
- "7.4": "php/php-32@=7.4.3301",
805
- },
806
- },
807
- "scripts": {"php"},
808
- "aliases": {},
809
- "env": {},
810
- },
811
- "bash": {
812
- "dependencies": {
813
- "latest": "wasmer/bash@=1.0.24",
814
- "8.3": "wasmer/bash@=1.0.24",
815
- },
816
- "scripts": {"bash", "sh"},
817
- "aliases": {},
818
- "env": {},
819
- },
820
- "static-web-server": {
821
- "dependencies": {
822
- "latest": "wasmer/static-web-server@=1.1.0",
823
- "2.38.0": "wasmer/static-web-server@=1.1.0",
824
- "0.1": "wasmer/static-web-server@=1.1.0",
825
- },
826
- "scripts": {"webserver"},
827
- "aliases": {"static-web-server": "webserver"},
828
- "env": {},
829
- },
830
- }
831
-
832
- def __init__(
833
- self,
834
- inner_builder: Builder,
835
- src_dir: Path,
836
- registry: Optional[str] = None,
837
- token: Optional[str] = None,
838
- bin: Optional[str] = None,
839
- ) -> None:
840
- self.src_dir = src_dir
841
- self.inner_builder = inner_builder
842
- # The path where we store the directory of the wasmer app in the inner builder
843
- self.wasmer_dir_path = self.src_dir / ".shipit" / "wasmer"
844
- self.wasmer_registry = registry
845
- self.wasmer_token = token
846
- self.bin = bin or "wasmer"
847
- self.default_env = {
848
- "SHIPIT_PYTHON_EXTRA_INDEX_URL": "https://pythonindex.wasix.org/simple",
849
- "SHIPIT_PYTHON_CROSS_PLATFORM": "wasix_wasm32",
850
- "SHIPIT_PYTHON_PRECOMPILE": "true",
851
- }
852
-
853
- def getenv(self, name: str) -> Optional[str]:
854
- return self.inner_builder.getenv(name) or self.default_env.get(name)
855
-
856
- def build(
857
- self, env: Dict[str, str], mounts: List[Mount], build: List[Step]
858
- ) -> None:
859
- return self.inner_builder.build(env, mounts, build)
860
-
861
- def build_prepare(self, serve: Serve) -> None:
862
- print("Building prepare")
863
- prepare_dir = self.wasmer_dir_path / "prepare"
864
- prepare_dir.mkdir(parents=True, exist_ok=True)
865
- env = serve.env or {}
866
- for dep in serve.deps:
867
- if dep.name in self.mapper:
868
- dep_env = self.mapper[dep.name].get("env")
869
- if dep_env is not None:
870
- env.update(dep_env)
871
- if env:
872
- env_lines = [f"export {k}={v}" for k, v in env.items()]
873
- env_lines = "\n".join(env_lines)
874
- else:
875
- env_lines = ""
876
-
877
- commands: List[str] = []
878
- if serve.cwd:
879
- commands.append(f"cd {serve.cwd}")
880
-
881
- if serve.prepare:
882
- for step in serve.prepare:
883
- if isinstance(step, RunStep):
884
- commands.append(step.command)
885
- elif isinstance(step, WorkdirStep):
886
- commands.append(f"cd {step.path}")
887
-
888
- body = "\n".join(filter(None, [env_lines, *commands]))
889
- content = f"#!/bin/bash\n\n{body}"
890
- console.print(
891
- f"\n[bold]Created prepare.sh script to run before packaging ✅[/bold]"
892
- )
893
- manifest_panel = Panel(
894
- Syntax(
895
- content,
896
- "bash",
897
- theme="monokai",
898
- background_color="default",
899
- line_numbers=True,
900
- ),
901
- box=box.SQUARE,
902
- border_style="bright_black",
903
- expand=False,
904
- )
905
- console.print(manifest_panel, markup=False, highlight=True)
906
-
907
- (prepare_dir / "prepare.sh").write_text(
908
- content,
909
- )
910
- (prepare_dir / "prepare.sh").chmod(0o755)
911
-
912
- def finalize_build(self, serve: Serve) -> None:
913
- inner = cast(Any, self.inner_builder)
914
- inner.finalize_build(serve)
915
-
916
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
917
- prepare_dir = self.wasmer_dir_path / "prepare"
918
- self.run_serve_command(
919
- "bash",
920
- extra_args=[
921
- f"--mapdir=/prepare:{prepare_dir}",
922
- "--",
923
- "/prepare/prepare.sh",
924
- ],
925
- )
926
-
927
- def build_serve(self, serve: Serve) -> None:
928
- from tomlkit import comment, document, nl, table, aot, string, array
929
-
930
- doc = document()
931
- doc.add(comment(f"Wasmer manifest generated with Shipit v{shipit_version}"))
932
- package = table()
933
- doc.add("package", package)
934
- package.add("entrypoint", "start")
935
- dependencies = table()
936
- doc.add("dependencies", dependencies)
937
-
938
- binaries = {}
939
-
940
- deps = serve.deps or []
941
- # We add bash if it's not present, as the prepare command is run in bash
942
- if serve.prepare:
943
- if not any(dep.name == "bash" for dep in deps):
944
- deps.append(Package("bash"))
945
-
946
- if deps:
947
- console.print(f"[bold]Mapping dependencies to Wasmer packages:[/bold]")
948
- for dep in deps:
949
- if dep.name in self.mapper:
950
- version = dep.version or "latest"
951
- mapped_dependencies = self.mapper[dep.name]["dependencies"]
952
- if dep.architecture:
953
- architecture_dependencies = (
954
- self.mapper[dep.name]
955
- .get("architecture_dependencies", {})
956
- .get(dep.architecture, {})
957
- )
958
- if architecture_dependencies:
959
- mapped_dependencies = architecture_dependencies
960
- if version in mapped_dependencies:
961
- console.print(
962
- f"* {dep.name}@{version} mapped to {self.mapper[dep.name]['dependencies'][version]}"
963
- )
964
- package_name, version = mapped_dependencies[version].split("@")
965
- dependencies.add(package_name, version)
966
- scripts = self.mapper[dep.name].get("scripts") or []
967
- for script in scripts:
968
- binaries[script] = {
969
- "script": f"{package_name}:{script}",
970
- "env": self.mapper[dep.name].get("env"),
971
- }
972
- aliases = self.mapper[dep.name].get("aliases") or {}
973
- for alias, script in aliases.items():
974
- binaries[alias] = {
975
- "script": f"{package_name}:{script}",
976
- "env": self.mapper[dep.name].get("env"),
977
- }
978
- else:
979
- raise Exception(
980
- f"Dependency {dep.name}@{version} not found in Wasmer"
981
- )
982
- else:
983
- raise Exception(f"Dependency {dep.name} not found in Wasmer")
984
-
985
- fs = table()
986
- doc.add("fs", fs)
987
- inner = cast(Any, self.inner_builder)
988
- if serve.mounts:
989
- for mount in serve.mounts:
990
- fs.add(
991
- str(mount.serve_path.absolute()),
992
- str(self.inner_builder.get_serve_mount_path(mount.name).absolute()),
993
- )
994
-
995
- doc.add(nl())
996
- if serve.commands:
997
- commands = aot()
998
- doc.add("command", commands)
999
- for command_name, command_line in serve.commands.items():
1000
- command = table()
1001
- commands.append(command)
1002
- parts = shlex.split(command_line)
1003
- program = parts[0]
1004
- command.add("name", command_name)
1005
- program_binary = binaries[program]
1006
- command.add("module", program_binary["script"])
1007
- command.add("runner", "wasi")
1008
- wasi_args = table()
1009
- if serve.cwd:
1010
- wasi_args.add("cwd", serve.cwd)
1011
- wasi_args.add("main-args", array(parts[1:]).multiline(True))
1012
- env = program_binary.get("env") or {}
1013
- if serve.env:
1014
- env.update(serve.env)
1015
- if env:
1016
- arr = array([f"{k}={v}" for k, v in env.items()]).multiline(True)
1017
- wasi_args.add("env", arr)
1018
- title = string("annotations.wasi", literal=False)
1019
- command.add(title, wasi_args)
1020
-
1021
- inner = cast(Any, self.inner_builder)
1022
- self.wasmer_dir_path.mkdir(parents=True, exist_ok=True)
1023
-
1024
- manifest = doc.as_string().replace(
1025
- '[command."annotations.wasi"]', "[command.annotations.wasi]"
1026
- )
1027
- console.print(f"\n[bold]Created wasmer.toml manifest ✅[/bold]")
1028
- manifest_panel = Panel(
1029
- Syntax(
1030
- manifest.strip(),
1031
- "toml",
1032
- theme="monokai",
1033
- background_color="default",
1034
- line_numbers=True,
1035
- ),
1036
- box=box.SQUARE,
1037
- border_style="bright_black",
1038
- expand=False,
1039
- )
1040
- console.print(manifest_panel, markup=False, highlight=True)
1041
- (self.wasmer_dir_path / "wasmer.toml").write_text(manifest)
1042
-
1043
- original_app_yaml_path = self.src_dir / "app.yaml"
1044
- if original_app_yaml_path.exists():
1045
- console.print(
1046
- f"[bold]Using original app.yaml found in source directory[/bold]"
1047
- )
1048
- yaml_config = yaml.safe_load(original_app_yaml_path.read_text())
1049
- else:
1050
- yaml_config = {
1051
- "kind": "wasmer.io/App.v0",
1052
- }
1053
- # Update the app to use the new package
1054
- yaml_config["package"] = "."
1055
- if serve.services:
1056
- capabilities = yaml_config.get("capabilities", {})
1057
- has_mysql = any(service.provider == "mysql" for service in serve.services)
1058
- # has_postgres = any(service.provider == "postgres" for service in serve.services)
1059
- # has_redis = any(service.provider == "redis" for service in serve.services)
1060
- if has_mysql:
1061
- capabilities["database"] = {"engine": "mysql"}
1062
- yaml_config["capabilities"] = capabilities
1063
-
1064
- # Attach declared volumes to the app manifest (serve-time mounts)
1065
- if serve.volumes:
1066
- volumes_yaml = yaml_config.get("volumes", [])
1067
- for vol in serve.volumes:
1068
- volumes_yaml.append(
1069
- {
1070
- "name": vol.name,
1071
- "mount": str(vol.serve_path),
1072
- }
1073
- )
1074
- yaml_config["volumes"] = volumes_yaml
1075
-
1076
- # If it has a php dependency, set the scaling mode to single_concurrency
1077
- has_php = any(dep.name == "php" for dep in serve.deps)
1078
- if has_php:
1079
- scaling = yaml_config.get("scaling", {})
1080
- scaling["mode"] = "single_concurrency"
1081
- yaml_config["scaling"] = scaling
1082
-
1083
- if "after_deploy" in serve.commands:
1084
- jobs = yaml_config.get("jobs", [])
1085
- jobs.append(
1086
- {
1087
- "name": "after_deploy",
1088
- "trigger": "post-deployment",
1089
- "action": {"execute": {"command": "after_deploy"}},
1090
- }
1091
- )
1092
- yaml_config["jobs"] = jobs
1093
-
1094
- app_yaml = yaml.dump(
1095
- yaml_config,
1096
- )
1097
-
1098
- console.print(f"\n[bold]Created app.yaml manifest ✅[/bold]")
1099
- app_yaml_panel = Panel(
1100
- Syntax(
1101
- app_yaml.strip(),
1102
- "yaml",
1103
- theme="monokai",
1104
- background_color="default",
1105
- line_numbers=True,
1106
- ),
1107
- box=box.SQUARE,
1108
- border_style="bright_black",
1109
- expand=False,
1110
- )
1111
- console.print(app_yaml_panel, markup=False, highlight=True)
1112
- (self.wasmer_dir_path / "app.yaml").write_text(app_yaml)
1113
-
1114
- # self.inner_builder.build_serve(serve)
1115
-
1116
- def run_serve_command(
1117
- self, command: str, extra_args: Optional[List[str]] = None
1118
- ) -> None:
1119
- console.print(f"\n[bold]Serving site[/bold]: running {command} command")
1120
- extra_args = extra_args or []
1121
-
1122
- if self.wasmer_registry:
1123
- extra_args = [f"--registry={self.wasmer_registry}"] + extra_args
1124
- self.run_command(
1125
- self.bin,
1126
- [
1127
- "run",
1128
- str(self.wasmer_dir_path.absolute()),
1129
- "--net",
1130
- f"--command={command}",
1131
- *extra_args,
1132
- ],
1133
- )
1134
-
1135
- def run_command(
1136
- self, command: str, extra_args: Optional[List[str]] | None = None
1137
- ) -> Any:
1138
- sh.Command(command)(
1139
- *(extra_args or []), _out=write_stdout, _err=write_stderr, _env=os.environ
1140
- )
1141
-
1142
- def deploy_config(self, config_path: Path) -> None:
1143
- package_webc_path = self.wasmer_dir_path / "package.webc"
1144
- app_yaml_path = self.wasmer_dir_path / "app.yaml"
1145
- package_webc_path.parent.mkdir(parents=True, exist_ok=True)
1146
- self.run_command(
1147
- self.bin,
1148
- ["package", "build", self.wasmer_dir_path, "--out", package_webc_path],
1149
- )
1150
- config_path.write_text(
1151
- json.dumps(
1152
- {
1153
- "app_yaml_path": str(app_yaml_path.absolute()),
1154
- "package_webc_path": str(package_webc_path.absolute()),
1155
- "package_webc_size": package_webc_path.stat().st_size,
1156
- "package_webc_sha256": hashlib.sha256(
1157
- package_webc_path.read_bytes()
1158
- ).hexdigest(),
1159
- }
1160
- )
1161
- )
1162
- console.print(f"\n[bold]Saved deploy config to {config_path}[/bold]")
1163
-
1164
- def deploy(
1165
- self, app_owner: Optional[str] = None, app_name: Optional[str] = None
1166
- ) -> str:
1167
- extra_args = []
1168
- if self.wasmer_registry:
1169
- extra_args += ["--registry", self.wasmer_registry]
1170
- if self.wasmer_token:
1171
- extra_args += ["--token", self.wasmer_token]
1172
- if app_owner:
1173
- extra_args += ["--owner", app_owner]
1174
- if app_name:
1175
- extra_args += ["--app-name", app_name]
1176
- # self.run_command(
1177
- # self.bin,
1178
- # [
1179
- # "package",
1180
- # "push",
1181
- # self.wasmer_dir_path,
1182
- # "--namespace",
1183
- # app_owner,
1184
- # "--non-interactive",
1185
- # *extra_args,
1186
- # ],
1187
- # )
1188
- return self.run_command(
1189
- self.bin,
1190
- [
1191
- "deploy",
1192
- "--publish-package",
1193
- "--dir",
1194
- self.wasmer_dir_path,
1195
- "--non-interactive",
1196
- *extra_args,
1197
- ],
1198
- )
49
+ serve_path: str
1199
50
 
1200
51
 
1201
52
  class Ctx:
1202
- def __init__(self, builder: Builder) -> None:
1203
- self.builder = builder
53
+ def __init__(self, build_backend: BuildBackend, runner: Runner) -> None:
54
+ self.build_backend = build_backend
55
+ self.runner = runner
1204
56
  self.packages: Dict[str, Package] = {}
1205
57
  self.builds: List[Build] = []
1206
58
  self.steps: List[Step] = []
@@ -1208,7 +60,6 @@ class Ctx:
1208
60
  self.mounts: List[Mount] = []
1209
61
  self.volumes: List[Volume] = []
1210
62
  self.services: Dict[str, Service] = {}
1211
- self.getenv_variables: Set[str] = set()
1212
63
 
1213
64
  def add_package(self, package: Package) -> str:
1214
65
  index = f"{package.name}@{package.version}" if package.version else package.name
@@ -1254,10 +105,6 @@ class Ctx:
1254
105
  self.steps.append(step)
1255
106
  return f"ref:step:{len(self.steps) - 1}"
1256
107
 
1257
- def getenv(self, name: str) -> Optional[str]:
1258
- self.getenv_variables.add(name)
1259
- return self.builder.getenv(name)
1260
-
1261
108
  def dep(
1262
109
  self,
1263
110
  name: str,
@@ -1306,9 +153,7 @@ class Ctx:
1306
153
  commands=commands,
1307
154
  prepare=prepare_steps,
1308
155
  workers=workers,
1309
- mounts=self.get_refs([mount["ref"] for mount in mounts])
1310
- if mounts
1311
- else None,
156
+ mounts=self.get_refs([mount.ref for mount in mounts]) if mounts else None,
1312
157
  volumes=self.get_refs([volume["ref"] for volume in volumes])
1313
158
  if volumes
1314
159
  else None,
@@ -1355,15 +200,16 @@ class Ctx:
1355
200
  return f"ref:mount:{len(self.mounts) - 1}"
1356
201
 
1357
202
  def mount(self, name: str) -> Optional[str]:
1358
- build_path = self.builder.get_build_mount_path(name)
1359
- serve_path = self.builder.get_serve_mount_path(name)
203
+ build_path = self.build_backend.get_build_mount_path(name)
204
+ serve_path = self.runner.get_serve_mount_path(name)
1360
205
  mount = Mount(name, build_path, serve_path)
1361
206
  ref = self.add_mount(mount)
1362
- return {
1363
- "ref": ref,
1364
- "build": str(build_path.absolute()),
1365
- "serve": str(serve_path.absolute()),
1366
- }
207
+
208
+ return CtxMount(
209
+ ref=ref,
210
+ path=str(build_path.absolute()),
211
+ serve_path=str(serve_path.absolute()),
212
+ )
1367
213
 
1368
214
  def add_volume(self, volume: Volume) -> Optional[str]:
1369
215
  self.volumes.append(volume)
@@ -1379,35 +225,92 @@ class Ctx:
1379
225
  }
1380
226
 
1381
227
 
1382
- def evaluate_shipit(shipit_file: Path, builder: Builder) -> Tuple[Ctx, Serve]:
228
+ def evaluate_shipit(
229
+ shipit_file: Path,
230
+ build_backend: BuildBackend,
231
+ runner: Runner,
232
+ provider_config: Config,
233
+ ) -> Tuple[Ctx, Serve]:
1383
234
  source = shipit_file.read_text()
1384
- ctx = Ctx(builder)
1385
- glb = sl.Globals.standard()
1386
- mod = sl.Module()
1387
-
1388
- mod.add_callable("service", ctx.service)
1389
- mod.add_callable("getenv", ctx.getenv)
1390
- mod.add_callable("dep", ctx.dep)
1391
- mod.add_callable("serve", ctx.serve)
1392
- mod.add_callable("run", ctx.run)
1393
- mod.add_callable("mount", ctx.mount)
1394
- mod.add_callable("volume", ctx.volume)
1395
- mod.add_callable("workdir", ctx.workdir)
1396
- mod.add_callable("copy", ctx.copy)
1397
- mod.add_callable("path", ctx.path)
1398
- mod.add_callable("env", ctx.env)
1399
- mod.add_callable("use", ctx.use)
1400
-
1401
- dialect = sl.Dialect.extended()
1402
- dialect.enable_f_strings = True
1403
-
1404
- ast = sl.parse("shipit", source, dialect=dialect)
1405
-
1406
- sl.eval(mod, ast, glb)
235
+ ctx = Ctx(build_backend, runner)
236
+ glb = sl.GlobalsBuilder.standard()
237
+
238
+ glb.set("PORT", str(provider_config.port or "8080"))
239
+ glb.set("config", provider_config)
240
+ glb.set("service", ctx.service)
241
+ glb.set("dep", ctx.dep)
242
+ glb.set("serve", ctx.serve)
243
+ glb.set("run", ctx.run)
244
+ glb.set("mount", ctx.mount)
245
+ glb.set("volume", ctx.volume)
246
+ glb.set("workdir", ctx.workdir)
247
+ glb.set("copy", ctx.copy)
248
+ glb.set("path", ctx.path)
249
+ glb.set("env", ctx.env)
250
+ glb.set("use", ctx.use)
251
+
252
+ dialect = sl.Dialect(enable_keyword_only_arguments=True, enable_f_strings=True)
253
+
254
+ ast = sl.AstModule.parse("Shipit", source, dialect=dialect)
255
+
256
+ evaluator = sl.Evaluator()
257
+ evaluator.eval_module(ast, glb.build())
1407
258
  if not ctx.serves:
1408
259
  raise ValueError(f"No serve definition found in {shipit_file}")
1409
260
  assert len(ctx.serves) <= 1, "Only one serve is allowed for now"
1410
261
  serve = next(iter(ctx.serves.values()))
262
+
263
+ # Now we apply the custom commands (start, after_deploy, build, install)
264
+ if provider_config.commands.start:
265
+ serve.commands["start"] = provider_config.commands.start
266
+
267
+ if provider_config.commands.after_deploy:
268
+ serve.commands["after_deploy"] = provider_config.commands.after_deploy
269
+
270
+ if provider_config.commands.build or provider_config.commands.install:
271
+ new_build = []
272
+ has_done_build = False
273
+ has_done_install = False
274
+ for step in serve.build:
275
+ if isinstance(step, RunStep):
276
+ if (
277
+ step.group == "build"
278
+ and not has_done_build
279
+ and provider_config.commands.build
280
+ ):
281
+ new_build.append(
282
+ RunStep(provider_config.commands.build, group="build")
283
+ )
284
+ has_done_build = True
285
+ elif (
286
+ step.group == "install"
287
+ and not has_done_install
288
+ and provider_config.commands.install
289
+ ):
290
+ new_build.append(
291
+ RunStep(provider_config.commands.install, group="install")
292
+ )
293
+ has_done_install = True
294
+ else:
295
+ new_build.append(step)
296
+ else:
297
+ new_build.append(step)
298
+ if not has_done_install and provider_config.commands.install:
299
+ new_build.append(RunStep(provider_config.commands.install, group="install"))
300
+ if not has_done_build and provider_config.commands.build:
301
+ new_build.append(RunStep(provider_config.commands.build, group="build"))
302
+ serve.build = new_build
303
+
304
+ if serve.commands.get("start"):
305
+ serve.commands["start"] = serve.commands["start"].replace(
306
+ "$PORT", str(provider_config.port or "8080")
307
+ )
308
+
309
+ if serve.commands.get("after_deploy"):
310
+ serve.commands["after_deploy"] = serve.commands["after_deploy"].replace(
311
+ "$PORT", str(provider_config.port or "8080")
312
+ )
313
+
1411
314
  return ctx, serve
1412
315
 
1413
316
 
@@ -1421,13 +324,6 @@ def print_help() -> None:
1421
324
  console.print(panel)
1422
325
 
1423
326
 
1424
- def download_file(url: str, path: Path) -> None:
1425
- response = requests.get(url)
1426
- response.raise_for_status()
1427
- path.parent.mkdir(parents=True, exist_ok=True)
1428
- path.write_bytes(response.content)
1429
-
1430
-
1431
327
  @app.command(name="auto")
1432
328
  def auto(
1433
329
  path: Path = typer.Argument(
@@ -1499,10 +395,6 @@ def auto(
1499
395
  None,
1500
396
  help="Name of the Wasmer app.",
1501
397
  ),
1502
- use_procfile: bool = typer.Option(
1503
- True,
1504
- help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
1505
- ),
1506
398
  install_command: Optional[str] = typer.Option(
1507
399
  None,
1508
400
  help="The install command to use (overwrites the default)",
@@ -1519,11 +411,22 @@ def auto(
1519
411
  None,
1520
412
  help="The environment to use (defaults to `.env`, it will use .env.<env_name> if provided)",
1521
413
  ),
1522
- use_provider: Optional[str] = typer.Option(
414
+ provider: Optional[str] = typer.Option(
1523
415
  None,
1524
416
  help="Use a specific provider to build the project.",
1525
417
  ),
418
+ config: Optional[str] = typer.Option(
419
+ None,
420
+ help="The JSON content to use as input.",
421
+ ),
422
+ serve_port: Optional[int] = typer.Option(
423
+ None,
424
+ help="The port to use (defaults to 8080).",
425
+ ),
1526
426
  ):
427
+ # We assume wasmer as an active flag if we pass wasmer deploy or wasmer deploy config
428
+ wasmer = wasmer or wasmer_deploy or (wasmer_deploy_config is not None)
429
+
1527
430
  if not path.exists():
1528
431
  raise Exception(f"The path {path} does not exist")
1529
432
 
@@ -1545,17 +448,20 @@ def auto(
1545
448
  generate(
1546
449
  path,
1547
450
  out=shipit_path,
1548
- use_procfile=use_procfile,
1549
451
  install_command=install_command,
1550
452
  build_command=build_command,
1551
453
  start_command=start_command,
1552
- use_provider=use_provider,
454
+ provider=provider,
455
+ config=config,
1553
456
  )
1554
457
 
1555
458
  build(
1556
459
  path,
1557
460
  shipit_path=shipit_path,
1558
- wasmer=(wasmer or wasmer_deploy),
461
+ install_command=install_command,
462
+ build_command=build_command,
463
+ start_command=start_command,
464
+ wasmer=wasmer,
1559
465
  docker=docker,
1560
466
  docker_client=docker_client,
1561
467
  skip_docker_if_safe_build=skip_docker_if_safe_build,
@@ -1564,6 +470,9 @@ def auto(
1564
470
  wasmer_bin=wasmer_bin,
1565
471
  skip_prepare=skip_prepare,
1566
472
  env_name=env_name,
473
+ serve_port=serve_port,
474
+ provider=provider,
475
+ config=config,
1567
476
  )
1568
477
  if start or wasmer_deploy or wasmer_deploy_config:
1569
478
  serve(
@@ -1598,10 +507,6 @@ def generate(
1598
507
  "--shipit-path",
1599
508
  help="Output path (defaults to the Shipit file in the provided path).",
1600
509
  ),
1601
- use_procfile: bool = typer.Option(
1602
- True,
1603
- help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
1604
- ),
1605
510
  install_command: Optional[str] = typer.Option(
1606
511
  None,
1607
512
  help="The install command to use (overwrites the default)",
@@ -1614,41 +519,50 @@ def generate(
1614
519
  None,
1615
520
  help="The start command to use (overwrites the default)",
1616
521
  ),
1617
- use_provider: Optional[str] = typer.Option(
522
+ provider: Optional[str] = typer.Option(
1618
523
  None,
1619
524
  help="Use a specific provider to build the project.",
1620
525
  ),
526
+ config: Optional[str] = typer.Option(
527
+ None,
528
+ help="The JSON content to use as input.",
529
+ ),
1621
530
  ):
1622
531
  if not path.exists():
1623
532
  raise Exception(f"The path {path} does not exist")
1624
533
 
1625
534
  if out is None:
1626
535
  out = path / "Shipit"
1627
- custom_commands = CustomCommands()
1628
- # if (path / "Dockerfile").exists():
1629
- # # We get the start command from the Dockerfile
1630
- # with open(path / "Dockerfile", "r") as f:
1631
- # cmd = None
1632
- # for line in f:
1633
- # if line.startswith("CMD "):
1634
- # cmd = line[4:].strip()
1635
- # cmd = json.loads(cmd)
1636
- # # We get the last command
1637
- # if cmd:
1638
- # if isinstance(cmd, list):
1639
- # cmd = " ".join(cmd)
1640
- # custom_commands.start = cmd
1641
- if use_procfile:
1642
- if (path / "Procfile").exists():
1643
- procfile = Procfile.loads((path / "Procfile").read_text())
1644
- custom_commands.start = procfile.get_start_command()
536
+
537
+ base_config = Config()
538
+ base_config.commands.enrich_from_path(path)
1645
539
  if start_command:
1646
- custom_commands.start = start_command
540
+ base_config.commands.start = start_command
1647
541
  if install_command:
1648
- custom_commands.install = install_command
542
+ base_config.commands.install = install_command
1649
543
  if build_command:
1650
- custom_commands.build = build_command
1651
- content = generate_shipit(path, custom_commands, use_provider=use_provider)
544
+ base_config.commands.build = build_command
545
+ provider_cls = load_provider(path, base_config, use_provider=provider)
546
+ provider_config = load_provider_config(
547
+ provider_cls, path, base_config, config=config
548
+ )
549
+ provider = provider_cls(path, provider_config)
550
+ content = generate_shipit(path, provider)
551
+ config_json = provider_config.model_dump_json(indent=2, exclude_defaults=True)
552
+ if config_json and config_json != "{}":
553
+ manifest_panel = Panel(
554
+ Syntax(
555
+ config_json,
556
+ "json",
557
+ theme="monokai",
558
+ background_color="default",
559
+ line_numbers=True,
560
+ ),
561
+ box=box.SQUARE,
562
+ border_style="bright_black",
563
+ expand=False,
564
+ )
565
+ console.print(manifest_panel, markup=False, highlight=True)
1652
566
  out.write_text(content)
1653
567
  console.print(f"[bold]Generated Shipit[/bold] at {out.absolute()}")
1654
568
 
@@ -1725,25 +639,40 @@ def serve(
1725
639
  help="Save the output of the Wasmer build to a json file",
1726
640
  ),
1727
641
  ) -> None:
642
+ # We assume wasmer as an active flag if we pass wasmer deploy or wasmer deploy config
643
+ wasmer = wasmer or wasmer_deploy or (wasmer_deploy_config is not None)
644
+
1728
645
  if not path.exists():
1729
646
  raise Exception(f"The path {path} does not exist")
1730
647
 
1731
- builder: Builder
1732
648
  if docker or docker_client:
1733
- builder = DockerBuilder(path, docker_client)
649
+ build_backend: BuildBackend = DockerBuildBackend(
650
+ path, ASSETS_PATH, docker_client
651
+ )
1734
652
  else:
1735
- builder = LocalBuilder(path)
1736
- if wasmer or wasmer_deploy or wasmer_deploy_config:
1737
- builder = WasmerBuilder(
1738
- builder, path, registry=wasmer_registry, token=wasmer_token, bin=wasmer_bin
653
+ build_backend = LocalBuildBackend(path, ASSETS_PATH)
654
+
655
+ if wasmer:
656
+ runner: Runner = WasmerRunner(
657
+ build_backend,
658
+ path,
659
+ registry=wasmer_registry,
660
+ token=wasmer_token,
661
+ bin=wasmer_bin,
1739
662
  )
663
+ else:
664
+ runner = LocalRunner(build_backend, path)
1740
665
 
1741
666
  if wasmer_deploy_config:
1742
- builder.deploy_config(wasmer_deploy_config)
667
+ if not isinstance(runner, WasmerRunner):
668
+ raise RuntimeError("--wasmer-deploy-config requires the Wasmer runner")
669
+ runner.deploy_config(wasmer_deploy_config)
1743
670
  elif wasmer_deploy:
1744
- builder.deploy(app_owner=wasmer_app_owner, app_name=wasmer_app_name)
671
+ if not isinstance(runner, WasmerRunner):
672
+ raise RuntimeError("--wasmer-deploy requires the Wasmer runner")
673
+ runner.deploy(app_owner=wasmer_app_owner, app_name=wasmer_app_name)
1745
674
  elif start:
1746
- builder.run_serve_command("start")
675
+ runner.run_serve_command("start")
1747
676
 
1748
677
 
1749
678
  @app.command(name="plan")
@@ -1796,10 +725,6 @@ def plan(
1796
725
  None,
1797
726
  help="Use a specific Docker client (such as depot, podman, etc.)",
1798
727
  ),
1799
- use_procfile: bool = typer.Option(
1800
- True,
1801
- help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
1802
- ),
1803
728
  install_command: Optional[str] = typer.Option(
1804
729
  None,
1805
730
  help="The install command to use (overwrites the default)",
@@ -1812,10 +737,18 @@ def plan(
1812
737
  None,
1813
738
  help="The start command to use (overwrites the default)",
1814
739
  ),
1815
- use_provider: Optional[str] = typer.Option(
740
+ provider: Optional[str] = typer.Option(
1816
741
  None,
1817
742
  help="Use a specific provider to build the project.",
1818
743
  ),
744
+ config: Optional[str] = typer.Option(
745
+ None,
746
+ help="The JSON content to use as input.",
747
+ ),
748
+ serve_port: Optional[int] = typer.Option(
749
+ None,
750
+ help="The port to use (defaults to 8080).",
751
+ ),
1819
752
  ) -> None:
1820
753
  if not path.exists():
1821
754
  raise Exception(f"The path {path} does not exist")
@@ -1838,39 +771,48 @@ def plan(
1838
771
  generate(
1839
772
  path,
1840
773
  out=shipit_path,
1841
- use_procfile=use_procfile,
1842
774
  install_command=install_command,
1843
775
  build_command=build_command,
1844
776
  start_command=start_command,
1845
- use_provider=use_provider,
777
+ provider=provider,
778
+ config=config,
1846
779
  )
1847
780
 
1848
- custom_commands = CustomCommands()
1849
- procfile_path = path / "Procfile"
1850
- if procfile_path.exists():
1851
- try:
1852
- procfile = Procfile.loads(procfile_path.read_text())
1853
- custom_commands.start = procfile.get_start_command()
1854
- except Exception:
1855
- pass
1856
-
1857
781
  shipit_file = get_shipit_path(path, shipit_path)
1858
782
 
1859
- builder: Builder
1860
783
  if docker or docker_client:
1861
- builder = DockerBuilder(path, docker_client)
784
+ build_backend: BuildBackend = DockerBuildBackend(
785
+ path, ASSETS_PATH, docker_client
786
+ )
1862
787
  else:
1863
- builder = LocalBuilder(path)
788
+ build_backend = LocalBuildBackend(path, ASSETS_PATH)
1864
789
  if wasmer:
1865
- builder = WasmerBuilder(
1866
- builder, path, registry=wasmer_registry, token=wasmer_token, bin=wasmer_bin
790
+ runner: Runner = WasmerRunner(
791
+ build_backend,
792
+ path,
793
+ registry=wasmer_registry,
794
+ token=wasmer_token,
795
+ bin=wasmer_bin,
1867
796
  )
797
+ else:
798
+ runner = LocalRunner(build_backend, path)
1868
799
 
1869
- ctx, serve = evaluate_shipit(shipit_file, builder)
1870
- metadata_commands: Dict[str, Optional[str]] = {
1871
- "start": serve.commands.get("start"),
1872
- "after_deploy": serve.commands.get("after_deploy"),
1873
- }
800
+ base_config = Config()
801
+ base_config.commands.enrich_from_path(path)
802
+ if install_command:
803
+ base_config.commands.install = install_command
804
+ if build_command:
805
+ base_config.commands.build = build_command
806
+ if start_command:
807
+ base_config.commands.start = start_command
808
+ if serve_port:
809
+ base_config.port = serve_port
810
+ provider_cls = load_provider(path, base_config, use_provider=provider)
811
+ provider_config = load_provider_config(
812
+ provider_cls, path, base_config, config=config
813
+ )
814
+ # provider_config = runner.prepare_config(provider_config)
815
+ ctx, serve = evaluate_shipit(shipit_file, build_backend, runner, provider_config)
1874
816
 
1875
817
  def _collect_group_commands(group: str) -> Optional[str]:
1876
818
  commands = [
@@ -1882,25 +824,21 @@ def plan(
1882
824
  return None
1883
825
  return " && ".join(commands)
1884
826
 
1885
- metadata_install = _collect_group_commands("install")
1886
- metadata_build = _collect_group_commands("build")
1887
- metadata_commands["install"] = metadata_install
1888
- metadata_commands["build"] = metadata_build
1889
- platform: Optional[str]
1890
- try:
1891
- provider_cls = detect_provider(path, custom_commands)
1892
- provider_instance = provider_cls(path, custom_commands)
1893
- provider_instance.initialize()
1894
- platform = provider_instance.platform()
1895
- except Exception:
1896
- platform = None
827
+ start_command = serve.commands.get("start")
828
+ after_deploy_command = serve.commands.get("after_deploy")
829
+ install_command = _collect_group_commands("install")
830
+ build_command = _collect_group_commands("build")
831
+ if start_command:
832
+ provider_config.commands.start = start_command
833
+ if after_deploy_command:
834
+ provider_config.commands.after_deploy = after_deploy_command
835
+ if install_command:
836
+ provider_config.commands.install = install_command
837
+ if build_command:
838
+ provider_config.commands.build = build_command
1897
839
  plan_output = {
1898
- "provider": serve.provider,
1899
- "metadata": {
1900
- "platform": platform,
1901
- "commands": metadata_commands,
1902
- },
1903
- "config": sorted(ctx.getenv_variables),
840
+ "provider": provider_cls.name(),
841
+ "config": json.loads(provider_config.model_dump_json(exclude_defaults=True)),
1904
842
  "services": [
1905
843
  {"name": svc.name, "provider": svc.provider}
1906
844
  for svc in (serve.services or [])
@@ -1927,6 +865,18 @@ def build(
1927
865
  None,
1928
866
  help="The path to the Shipit file (defaults to Shipit in the provided path).",
1929
867
  ),
868
+ start_command: Optional[str] = typer.Option(
869
+ None,
870
+ help="The start command to use (overwrites the default)",
871
+ ),
872
+ install_command: Optional[str] = typer.Option(
873
+ None,
874
+ help="The install command to use (overwrites the default)",
875
+ ),
876
+ build_command: Optional[str] = typer.Option(
877
+ None,
878
+ help="The build command to use (overwrites the default)",
879
+ ),
1930
880
  wasmer: bool = typer.Option(
1931
881
  False,
1932
882
  help="Use Wasmer to build and serve the project.",
@@ -1963,23 +913,59 @@ def build(
1963
913
  None,
1964
914
  help="The environment to use (defaults to `.env`, it will use .env.<env_name> if provided)",
1965
915
  ),
916
+ serve_port: Optional[int] = typer.Option(
917
+ None,
918
+ help="The port to use (defaults to 8080).",
919
+ ),
920
+ provider: Optional[str] = typer.Option(
921
+ None,
922
+ help="Use a specific provider to build the project.",
923
+ ),
924
+ config: Optional[str] = typer.Option(
925
+ None,
926
+ help="The JSON content to use as input.",
927
+ ),
1966
928
  ) -> None:
1967
929
  if not path.exists():
1968
930
  raise Exception(f"The path {path} does not exist")
1969
931
 
1970
932
  shipit_file = get_shipit_path(path, shipit_path)
1971
933
 
1972
- builder: Builder
1973
934
  if docker or docker_client:
1974
- builder = DockerBuilder(path, docker_client)
935
+ build_backend: BuildBackend = DockerBuildBackend(
936
+ path, ASSETS_PATH, docker_client
937
+ )
1975
938
  else:
1976
- builder = LocalBuilder(path)
939
+ build_backend = LocalBuildBackend(path, ASSETS_PATH)
1977
940
  if wasmer:
1978
- builder = WasmerBuilder(
1979
- builder, path, registry=wasmer_registry, token=wasmer_token, bin=wasmer_bin
941
+ runner: Runner = WasmerRunner(
942
+ build_backend,
943
+ path,
944
+ registry=wasmer_registry,
945
+ token=wasmer_token,
946
+ bin=wasmer_bin,
1980
947
  )
948
+ else:
949
+ runner = LocalRunner(build_backend, path)
1981
950
 
1982
- ctx, serve = evaluate_shipit(shipit_file, builder)
951
+ base_config = Config()
952
+ base_config.commands.enrich_from_path(path)
953
+ if start_command:
954
+ base_config.commands.start = start_command
955
+ if install_command:
956
+ base_config.commands.install = install_command
957
+ if build_command:
958
+ base_config.commands.build = build_command
959
+ serve_port = serve_port or os.environ.get("PORT")
960
+ if serve_port:
961
+ base_config.port = serve_port
962
+
963
+ provider_cls = load_provider(path, base_config, use_provider=provider)
964
+ provider_config = load_provider_config(
965
+ provider_cls, path, base_config, config=config
966
+ )
967
+ provider_config = runner.prepare_config(provider_config)
968
+ ctx, serve = evaluate_shipit(shipit_file, build_backend, runner, provider_config)
1983
969
  env = {
1984
970
  "PATH": "",
1985
971
  "COLORTERM": os.environ.get("COLORTERM", ""),
@@ -1999,6 +985,9 @@ def build(
1999
985
  return build(
2000
986
  path,
2001
987
  shipit_path=shipit_path,
988
+ install_command=install_command,
989
+ build_command=build_command,
990
+ start_command=start_command,
2002
991
  wasmer=wasmer,
2003
992
  skip_prepare=skip_prepare,
2004
993
  wasmer_bin=wasmer_bin,
@@ -2008,6 +997,9 @@ def build(
2008
997
  docker_client=None,
2009
998
  skip_docker_if_safe_build=False,
2010
999
  env_name=env_name,
1000
+ serve_port=serve_port,
1001
+ provider=provider,
1002
+ config=config,
2011
1003
  )
2012
1004
 
2013
1005
  serve.env = serve.env or {}
@@ -2019,14 +1011,15 @@ def build(
2019
1011
  env_vars = dotenv_values(path / f".env.{env_name}")
2020
1012
  serve.env.update(env_vars)
2021
1013
 
1014
+ assert serve.commands.get("start"), (
1015
+ "No start command could be found, please provide a start command"
1016
+ )
1017
+
2022
1018
  # Build and serve
2023
- builder.build(env, serve.mounts, serve.build)
2024
- if serve.prepare:
2025
- builder.build_prepare(serve)
2026
- builder.build_serve(serve)
2027
- builder.finalize_build(serve)
1019
+ build_backend.build(serve.name, env, serve.mounts or [], serve.build)
1020
+ runner.build(serve)
2028
1021
  if serve.prepare and not skip_prepare:
2029
- builder.prepare(env, serve.prepare)
1022
+ runner.prepare(env, serve.prepare)
2030
1023
 
2031
1024
 
2032
1025
  def get_shipit_path(path: Path, shipit_path: Optional[Path] = None) -> Path: