shipit-cli 0.14.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
shipit/cli.py CHANGED
@@ -1,47 +1,40 @@
1
1
  import tempfile
2
- import hashlib
3
- import requests
4
2
  import os
5
- import shlex
6
- import shutil
7
3
  import sys
8
4
  import json
9
- import yaml
10
- import base64
11
- from dataclasses import dataclass
12
5
  from pathlib import Path
13
- from typing import (
14
- Any,
15
- Dict,
16
- List,
17
- Optional,
18
- Protocol,
19
- Set,
20
- Tuple,
21
- TypedDict,
22
- Union,
23
- Literal,
24
- cast,
25
- )
26
- from shutil import copy, copytree, ignore_patterns
6
+ from dataclasses import dataclass
7
+ from typing import Any, Dict, List, Optional, Tuple, cast, Literal
27
8
 
28
- import sh # type: ignore[import-untyped]
29
- import starlark as sl
9
+ import xingque as sl
30
10
  import typer
31
11
  from rich import box
32
- from rich.console import Console
33
12
  from rich.panel import Panel
34
- from rich.rule import Rule
35
13
  from rich.syntax import Syntax
36
14
 
37
- from shipit.version import version as shipit_version
38
- from shipit.generator import generate_shipit, detect_provider
39
- from shipit.providers.base import CustomCommands
40
- from shipit.procfile import Procfile
15
+ from shipit.generator import generate_shipit, load_provider, load_provider_config
16
+ from shipit.providers.base import Config
41
17
  from dotenv import dotenv_values
42
-
43
-
44
- console = Console()
18
+ from shipit.builders import BuildBackend, DockerBuildBackend, LocalBuildBackend
19
+ from shipit.runners import Runner, LocalRunner, WasmerRunner
20
+ from shipit.shipit_types import (
21
+ Build,
22
+ CopyStep,
23
+ EnvStep,
24
+ Mount,
25
+ Package,
26
+ PathStep,
27
+ PrepareStep,
28
+ RunStep,
29
+ Serve,
30
+ Service,
31
+ Step,
32
+ UseStep,
33
+ Volume,
34
+ WorkdirStep,
35
+ )
36
+ from shipit.ui import console
37
+ from shipit.version import version as shipit_version
45
38
 
46
39
  app = typer.Typer(invoke_without_command=True)
47
40
 
@@ -50,1157 +43,16 @@ ASSETS_PATH = DIR_PATH / "assets"
50
43
 
51
44
 
52
45
  @dataclass
53
- class Mount:
54
- name: str
55
- build_path: Path
56
- serve_path: Path
57
-
58
-
59
- @dataclass
60
- class Volume:
61
- name: str
62
- serve_path: Path
63
-
64
-
65
- @dataclass
66
- class Service:
67
- name: str
68
- provider: Literal[
69
- "postgres", "mysql", "redis"
70
- ] # Right now we only support postgres and mysql
71
-
72
-
73
- @dataclass
74
- class Serve:
75
- name: str
76
- provider: str
77
- build: List["Step"]
78
- deps: List["Package"]
79
- commands: Dict[str, str]
80
- cwd: Optional[str] = None
81
- prepare: Optional[List["PrepareStep"]] = None
82
- workers: Optional[List[str]] = None
83
- mounts: Optional[List[Mount]] = None
84
- volumes: Optional[List[Volume]] = None
85
- env: Optional[Dict[str, str]] = None
86
- services: Optional[List[Service]] = None
87
-
88
-
89
- @dataclass
90
- class Package:
91
- name: str
92
- version: Optional[str] = None
93
- architecture: Optional[Literal["64-bit", "32-bit"]] = None
94
-
95
- def __str__(self) -> str: # pragma: no cover - simple representation
96
- name = f"{self.name}({self.architecture})" if self.architecture else self.name
97
- if self.version is None:
98
- return name
99
- return f"{name}@{self.version}"
100
-
101
-
102
- @dataclass
103
- class RunStep:
104
- command: str
105
- inputs: Optional[List[str]] = None
106
- outputs: Optional[List[str]] = None
107
- group: Optional[str] = None
108
-
109
-
110
- @dataclass
111
- class WorkdirStep:
112
- path: Path
113
-
114
-
115
- @dataclass
116
- class CopyStep:
117
- source: str
118
- target: str
119
- ignore: Optional[List[str]] = None
120
- # We can copy from the app source or from the shipit assets folder
121
- base: Literal["source", "assets"] = "source"
122
-
123
- def is_download(self) -> bool:
124
- return self.source.startswith("http://") or self.source.startswith("https://")
125
-
126
-
127
- @dataclass
128
- class EnvStep:
129
- variables: Dict[str, str]
130
-
131
- def __str__(self) -> str: # pragma: no cover - simple representation
132
- return " ".join([f"{key}={value}" for key, value in self.variables.items()])
133
-
134
-
135
- @dataclass
136
- class UseStep:
137
- dependencies: List[Package]
138
-
139
-
140
- @dataclass
141
- class PathStep:
46
+ class CtxMount:
47
+ ref: str
142
48
  path: str
143
-
144
-
145
- Step = Union[RunStep, CopyStep, EnvStep, PathStep, UseStep, WorkdirStep]
146
- PrepareStep = Union[RunStep]
147
-
148
-
149
- @dataclass
150
- class Build:
151
- deps: List[Package]
152
- steps: List[Step]
153
-
154
-
155
- def write_stdout(line: str) -> None:
156
- sys.stdout.write(line) # print to console
157
- sys.stdout.flush()
158
-
159
-
160
- def write_stderr(line: str) -> None:
161
- sys.stderr.write(line) # print to console
162
- sys.stderr.flush()
163
-
164
-
165
- class MapperItem(TypedDict):
166
- dependencies: Dict[str, str]
167
- scripts: Set[str]
168
- env: Dict[str, str]
169
- aliases: Dict[str, str]
170
-
171
-
172
- class Builder(Protocol):
173
- def build(
174
- self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
175
- ) -> None: ...
176
- def build_prepare(self, serve: Serve) -> None: ...
177
- def build_serve(self, serve: Serve) -> None: ...
178
- def finalize_build(self, serve: Serve) -> None: ...
179
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None: ...
180
- def getenv(self, name: str) -> Optional[str]: ...
181
- def run_serve_command(self, command: str) -> None: ...
182
- def run_command(
183
- self, command: str, extra_args: Optional[List[str]] | None = None
184
- ) -> Any: ...
185
- def get_build_mount_path(self, name: str) -> Path: ...
186
- def get_serve_mount_path(self, name: str) -> Path: ...
187
-
188
-
189
- class DockerBuilder:
190
- mise_mapper = {
191
- "php": {
192
- "source": "ubi:adwinying/php",
193
- },
194
- "composer": {
195
- "source": "ubi:composer/composer",
196
- "postinstall": """composer_dir=$(mise where ubi:composer/composer); ln -s "$composer_dir/composer.phar" /usr/local/bin/composer""",
197
- },
198
- }
199
-
200
- def __init__(self, src_dir: Path, docker_client: Optional[str] = None) -> None:
201
- self.src_dir = src_dir
202
- self.docker_file_contents = ""
203
- self.docker_path = self.src_dir / ".shipit" / "docker"
204
- self.docker_out_path = self.docker_path / "out"
205
- self.depot_metadata = self.docker_path / "depot-build.json"
206
- self.docker_file_path = self.docker_path / "Dockerfile"
207
- self.docker_name_path = self.docker_path / "name"
208
- self.docker_ignore_path = self.docker_path / "Dockerfile.dockerignore"
209
- self.shipit_docker_path = Path("/shipit")
210
- self.docker_client = docker_client or "docker"
211
- self.env = {
212
- "HOME": "/root",
213
- }
214
-
215
- def get_mount_path(self, name: str) -> Path:
216
- if name == "app":
217
- return Path("app")
218
- else:
219
- return Path("opt") / name
220
-
221
- def get_build_mount_path(self, name: str) -> Path:
222
- path = Path("/") / self.get_mount_path(name)
223
- return path
224
-
225
- def get_serve_mount_path(self, name: str) -> Path:
226
- return self.docker_out_path / self.get_mount_path(name)
227
-
228
- @property
229
- def is_depot(self) -> bool:
230
- return self.docker_client == "depot"
231
-
232
- def getenv(self, name: str) -> Optional[str]:
233
- return self.env.get(name) or os.environ.get(name)
234
-
235
- def mkdir(self, path: Path) -> Path:
236
- path = self.shipit_docker_path / path
237
- self.docker_file_contents += f"RUN mkdir -p {str(path.absolute())}\n"
238
- return path.absolute()
239
-
240
- def build_dockerfile(self, image_name: str) -> None:
241
- self.docker_file_path.write_text(self.docker_file_contents)
242
- self.docker_name_path.write_text(image_name)
243
- self.print_dockerfile()
244
- extra_args = []
245
- # if self.is_depot:
246
- # # We load the docker image back into the local docker daemon
247
- # # extra_args += ["--load"]
248
- # extra_args += ["--save", f"--metadata-file={self.depot_metadata.absolute()}"]
249
- sh.Command(self.docker_client)(
250
- "build",
251
- "-f",
252
- (self.docker_path / "Dockerfile").absolute(),
253
- "-t",
254
- image_name,
255
- "--platform",
256
- "linux/amd64",
257
- "--output",
258
- self.docker_out_path.absolute(),
259
- ".",
260
- *extra_args,
261
- _cwd=self.src_dir.absolute(),
262
- _env=os.environ, # Pass the current environment variables to the Docker client
263
- _out=write_stdout,
264
- _err=write_stderr,
265
- )
266
- # if self.is_depot:
267
- # json_text = self.depot_metadata.read_text()
268
- # json_data = json.loads(json_text)
269
- # build_data = json_data["depot.build"]
270
- # image_id = build_data["buildID"]
271
- # project = build_data["projectID"]
272
- # sh.Command("depot")(
273
- # "pull",
274
- # "--platform",
275
- # "linux/amd64",
276
- # "--project",
277
- # project,
278
- # image_id,
279
- # _cwd=self.src_dir.absolute(),
280
- # _env=os.environ, # Pass the current environment variables to the Docker client
281
- # _out=write_stdout,
282
- # _err=write_stderr,
283
- # )
284
- # # console.print(f"[bold]Image ID:[/bold] {image_id}")
285
-
286
- def finalize_build(self, serve: Serve) -> None:
287
- console.print(f"\n[bold]Building Docker file[/bold]")
288
- self.build_dockerfile(serve.name)
289
- console.print(Rule(characters="-", style="bright_black"))
290
- console.print(f"[bold]Build complete ✅[/bold]")
291
-
292
- def run_command(self, command: str, extra_args: Optional[List[str]] = None) -> Any:
293
- image_name = self.docker_name_path.read_text()
294
- docker_args: List[str] = [
295
- "run",
296
- "-p",
297
- "80:80",
298
- "--rm",
299
- ]
300
- # Attach volumes if present
301
- # if serve.volumes:
302
- # for vol in serve.volumes:
303
- # docker_args += [
304
- # "--mount",
305
- # f"type=volume,source={vol.name},target={str(vol.serve_path)}",
306
- # ]
307
- return sh.Command("docker")(
308
- *docker_args,
309
- image_name,
310
- command,
311
- *(extra_args or []),
312
- _env={
313
- "DOCKER_BUILDKIT": "1",
314
- **os.environ,
315
- }, # Pass the current environment variables to the Docker client
316
- _out=write_stdout,
317
- _err=write_stderr,
318
- )
319
-
320
- def create_file(self, path: Path, content: str, mode: int = 0o755) -> Path:
321
- # docker_files = self.docker_path / "files" / path.name
322
- # docker_files.write_text(content)
323
- # docker_files.chmod(mode)
324
- self.docker_file_contents += f"""
325
- RUN cat > {path.absolute()} <<'EOF'
326
- {content}
327
- EOF
328
-
329
- RUN chmod {oct(mode)[2:]} {path.absolute()}
330
- """
331
-
332
- return path.absolute()
333
-
334
- def print_dockerfile(self) -> None:
335
- docker_file = self.docker_path / "Dockerfile"
336
- manifest_panel = Panel(
337
- Syntax(
338
- docker_file.read_text(),
339
- "dockerfile",
340
- theme="monokai",
341
- background_color="default",
342
- line_numbers=True,
343
- ),
344
- box=box.SQUARE,
345
- border_style="bright_black",
346
- expand=False,
347
- )
348
- console.print(manifest_panel, markup=False, highlight=True)
349
-
350
- def add_dependency(self, dependency: Package):
351
- if dependency.name == "pie":
352
- self.docker_file_contents += f"RUN apt-get update && apt-get -y --no-install-recommends install gcc make autoconf libtool bison re2c pkg-config libpq-dev\n"
353
- self.docker_file_contents += f"RUN curl -L --output /usr/bin/pie https://github.com/php/pie/releases/download/1.2.0/pie.phar && chmod +x /usr/bin/pie\n"
354
- return
355
- elif dependency.name == "static-web-server":
356
- if dependency.version:
357
- self.docker_file_contents += (
358
- f"ENV SWS_INSTALL_VERSION={dependency.version}\n"
359
- )
360
- self.docker_file_contents += f"RUN curl --proto '=https' --tlsv1.2 -sSfL https://get.static-web-server.net | sh\n"
361
- return
362
-
363
- mapped_dependency = self.mise_mapper.get(dependency.name, {})
364
- package_name = mapped_dependency.get("source", dependency.name)
365
- if dependency.version:
366
- self.docker_file_contents += (
367
- f"RUN mise use --global {package_name}@{dependency.version}\n"
368
- )
369
- else:
370
- self.docker_file_contents += f"RUN mise use --global {package_name}\n"
371
- if mapped_dependency.get("postinstall"):
372
- self.docker_file_contents += f"RUN {mapped_dependency.get('postinstall')}\n"
373
-
374
- def build(
375
- self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
376
- ) -> None:
377
- base_path = self.docker_path
378
- shutil.rmtree(base_path, ignore_errors=True)
379
- base_path.mkdir(parents=True, exist_ok=True)
380
- self.docker_file_contents = "# syntax=docker/dockerfile:1.7-labs\n"
381
- self.docker_file_contents += "FROM debian:trixie-slim AS build\n"
382
-
383
- self.docker_file_contents += """
384
- RUN apt-get update \\
385
- && apt-get -y --no-install-recommends install \\
386
- build-essential gcc make autoconf libtool bison \\
387
- dpkg-dev pkg-config re2c locate \\
388
- libmariadb-dev libmariadb-dev-compat libpq-dev \\
389
- libvips-dev default-libmysqlclient-dev libmagickwand-dev \\
390
- libicu-dev libxml2-dev libxslt-dev libyaml-dev \\
391
- sudo curl ca-certificates \\
392
- && rm -rf /var/lib/apt/lists/*
393
-
394
- SHELL ["/bin/bash", "-o", "pipefail", "-c"]
395
- ENV MISE_DATA_DIR="/mise"
396
- ENV MISE_CONFIG_DIR="/mise"
397
- ENV MISE_CACHE_DIR="/mise/cache"
398
- ENV MISE_INSTALL_PATH="/usr/local/bin/mise"
399
- ENV PATH="/mise/shims:$PATH"
400
- # ENV MISE_VERSION="..."
401
-
402
- RUN curl https://mise.run | sh
403
- """
404
- # docker_file_contents += "RUN curl https://mise.run | sh\n"
405
- # self.docker_file_contents += """
406
- # RUN curl https://get.wasmer.io -sSfL | sh -s "v6.1.0-rc.3"
407
- # ENV PATH="/root/.wasmer/bin:${PATH}"
408
- # """
409
- for mount in mounts:
410
- self.docker_file_contents += f"RUN mkdir -p {mount.build_path.absolute()}\n"
411
-
412
- for step in steps:
413
- if isinstance(step, WorkdirStep):
414
- self.docker_file_contents += f"WORKDIR {step.path.absolute()}\n"
415
- elif isinstance(step, RunStep):
416
- if step.inputs:
417
- pre = "\\\n " + "".join(
418
- [
419
- f"--mount=type=bind,source={input},target={input} \\\n "
420
- for input in step.inputs
421
- ]
422
- )
423
- else:
424
- pre = ""
425
- self.docker_file_contents += f"RUN {pre}{step.command}\n"
426
- elif isinstance(step, CopyStep):
427
- if step.is_download():
428
- self.docker_file_contents += (
429
- "ADD " + step.source + " " + step.target + "\n"
430
- )
431
- elif step.base == "assets":
432
- # Detect if the asset exists and is a file
433
- if (ASSETS_PATH / step.source).is_file():
434
- # Read the file content and write it to the target file
435
- content_base64 = base64.b64encode(
436
- (ASSETS_PATH / step.source).read_bytes()
437
- ).decode("utf-8")
438
- self.docker_file_contents += (
439
- f"RUN echo '{content_base64}' | base64 -d > {step.target}\n"
440
- )
441
- elif (ASSETS_PATH / step.source).is_dir():
442
- raise Exception(
443
- f"Asset {step.source} is a directory, shipit doesn't currently support coppying assets directories inside Docker"
444
- )
445
- else:
446
- raise Exception(f"Asset {step.source} does not exist")
447
- else:
448
- if step.ignore:
449
- exclude = (
450
- " \\\n"
451
- + " \\\n".join(
452
- [f" --exclude={ignore}" for ignore in step.ignore]
453
- )
454
- + " \\\n "
455
- )
456
- else:
457
- exclude = ""
458
- self.docker_file_contents += (
459
- f"COPY{exclude} {step.source} {step.target}\n"
460
- )
461
- elif isinstance(step, EnvStep):
462
- env_vars = " ".join(
463
- [f"{key}={value}" for key, value in step.variables.items()]
464
- )
465
- self.docker_file_contents += f"ENV {env_vars}\n"
466
- elif isinstance(step, PathStep):
467
- self.docker_file_contents += f"ENV PATH={step.path}:$PATH\n"
468
- elif isinstance(step, UseStep):
469
- for dependency in step.dependencies:
470
- self.add_dependency(dependency)
471
-
472
- self.docker_file_contents += """
473
- FROM scratch
474
- """
475
- for mount in mounts:
476
- self.docker_file_contents += (
477
- f"COPY --from=build {mount.build_path} {mount.build_path}\n"
478
- )
479
-
480
- self.docker_ignore_path.write_text("""
481
- .shipit
482
- Shipit
483
- """)
484
-
485
- def get_path(self) -> Path:
486
- return Path("/")
487
-
488
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
489
- raise NotImplementedError
490
-
491
- def build_serve(self, serve: Serve) -> None:
492
- serve_command_path = self.mkdir(Path("serve") / "bin")
493
- console.print(f"[bold]Serve Commands:[/bold]")
494
- for dep in serve.deps:
495
- self.add_dependency(dep)
496
-
497
- for command in serve.commands:
498
- console.print(f"* {command}")
499
- command_path = serve_command_path / command
500
- self.create_file(
501
- command_path,
502
- f"#!/bin/bash\ncd {serve.cwd}\n{serve.commands[command]}",
503
- mode=0o755,
504
- )
505
-
506
- def run_serve_command(self, command: str) -> None:
507
- path = self.shipit_docker_path / "serve" / "bin" / command
508
- self.run_command(str(path))
509
-
510
-
511
- class LocalBuilder:
512
- def __init__(self, src_dir: Path) -> None:
513
- self.src_dir = src_dir
514
- self.local_path = self.src_dir / ".shipit" / "local"
515
- self.serve_bin_path = self.local_path / "serve" / "bin"
516
- self.prepare_bash_script = self.local_path / "prepare" / "prepare.sh"
517
- self.build_path = self.local_path / "build"
518
- self.workdir = self.build_path
519
-
520
- def get_mount_path(self, name: str) -> Path:
521
- if name == "app":
522
- return self.build_path / "app"
523
- else:
524
- return self.build_path / "opt" / name
525
-
526
- def get_build_mount_path(self, name: str) -> Path:
527
- return self.get_mount_path(name)
528
-
529
- def get_serve_mount_path(self, name: str) -> Path:
530
- return self.get_mount_path(name)
531
-
532
- def execute_step(self, step: Step, env: Dict[str, str]) -> None:
533
- build_path = self.workdir
534
- if isinstance(step, UseStep):
535
- console.print(
536
- f"[bold]Using dependencies:[/bold] {', '.join([str(dep) for dep in step.dependencies])}"
537
- )
538
- elif isinstance(step, WorkdirStep):
539
- console.print(f"[bold]Working in {step.path}[/bold]")
540
- self.workdir = step.path
541
- # We make sure the dir exists
542
- step.path.mkdir(parents=True, exist_ok=True)
543
- elif isinstance(step, RunStep):
544
- extra = ""
545
- if step.inputs:
546
- for input in step.inputs:
547
- print(f"Copying {input} to {build_path / input}")
548
- copy((self.src_dir / input), (build_path / input))
549
- all_inputs = ", ".join(step.inputs)
550
- extra = f" [bright_black]# using {all_inputs}[/bright_black]"
551
- console.print(
552
- f"[bright_black]$[/bright_black] [bold]{step.command}[/bold]{extra}"
553
- )
554
- command_line = step.command
555
- parts = shlex.split(command_line)
556
- program = parts[0]
557
- extended_paths = [
558
- str(build_path / path) for path in env["PATH"].split(os.pathsep)
559
- ]
560
- extended_paths.append(os.environ["PATH"])
561
- PATH = os.pathsep.join(extended_paths) # type: ignore
562
- exe = shutil.which(program, path=PATH)
563
- if not exe:
564
- raise Exception(f"Program is not installed: {program}")
565
- cmd = sh.Command("bash") # "grep"
566
- result = cmd(
567
- "-c",
568
- command_line,
569
- _env={**env, "PATH": PATH},
570
- _cwd=build_path,
571
- _out=write_stdout,
572
- _err=write_stderr,
573
- )
574
- elif isinstance(step, CopyStep):
575
- ignore_extra = ""
576
- if step.ignore:
577
- ignore_extra = (
578
- f" [bright_black]# ignoring {', '.join(step.ignore)}[/bright_black]"
579
- )
580
- ignore_matches = step.ignore if step.ignore else []
581
- ignore_matches.append(".shipit")
582
- ignore_matches.append("Shipit")
583
-
584
- if step.is_download():
585
- console.print(
586
- f"[bold]Download from {step.source} to {step.target}[/bold]"
587
- )
588
- download_file(step.source, (build_path / step.target))
589
- else:
590
- if step.base == "source":
591
- base = self.src_dir
592
- elif step.base == "assets":
593
- base = ASSETS_PATH
594
- else:
595
- raise Exception(f"Unknown base: {step.base}")
596
-
597
- console.print(
598
- f"[bold]Copy to {step.target} from {step.source}[/bold]{ignore_extra}"
599
- )
600
-
601
- if (base / step.source).is_dir():
602
- copytree(
603
- (base / step.source),
604
- (build_path / step.target),
605
- dirs_exist_ok=True,
606
- ignore=ignore_patterns(*ignore_matches),
607
- )
608
- elif (base / step.source).is_file():
609
- copy(
610
- (base / step.source),
611
- (build_path / step.target),
612
- )
613
- else:
614
- raise Exception(f"Source {step.source} is not a file or directory")
615
- elif isinstance(step, EnvStep):
616
- print(f"Setting environment variables: {step}")
617
- env.update(step.variables)
618
- elif isinstance(step, PathStep):
619
- console.print(f"[bold]Add {step.path}[/bold] to PATH")
620
- fullpath = step.path
621
- env["PATH"] = f"{fullpath}{os.pathsep}{env['PATH']}"
622
- else:
623
- raise Exception(f"Unknown step type: {type(step)}")
624
-
625
- def build(
626
- self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
627
- ) -> None:
628
- console.print(f"\n[bold]Building... 🚀[/bold]")
629
- base_path = self.local_path
630
- shutil.rmtree(base_path, ignore_errors=True)
631
- base_path.mkdir(parents=True, exist_ok=True)
632
- self.build_path.mkdir(exist_ok=True)
633
- for mount in mounts:
634
- mount.build_path.mkdir(parents=True, exist_ok=True)
635
- for step in steps:
636
- console.print(Rule(characters="-", style="bright_black"))
637
- self.execute_step(step, env)
638
-
639
- if "PATH" in env:
640
- path = base_path / ".path"
641
- path.write_text(env["PATH"]) # type: ignore
642
-
643
- console.print(Rule(characters="-", style="bright_black"))
644
- console.print(f"[bold]Build complete ✅[/bold]")
645
-
646
- def mkdir(self, path: Path) -> Path:
647
- path = self.get_path() / path
648
- path.mkdir(parents=True, exist_ok=True)
649
- return path.absolute()
650
-
651
- def create_file(self, path: Path, content: str, mode: int = 0o755) -> Path:
652
- path.write_text(content)
653
- path.chmod(mode)
654
- return path.absolute()
655
-
656
- def run_command(self, command: str, extra_args: Optional[List[str]] = None) -> Any:
657
- return sh.Command(command)(
658
- *(extra_args or []),
659
- _out=write_stdout,
660
- _err=write_stderr,
661
- _env=os.environ,
662
- )
663
-
664
- def getenv(self, name: str) -> Optional[str]:
665
- return os.environ.get(name)
666
-
667
- def get_path(self) -> Path:
668
- return self.local_path
669
-
670
- def build_prepare(self, serve: Serve) -> None:
671
- self.prepare_bash_script.parent.mkdir(parents=True, exist_ok=True)
672
- commands: List[str] = []
673
- if serve.cwd:
674
- commands.append(f"cd {serve.cwd}")
675
- if serve.prepare:
676
- for step in serve.prepare:
677
- if isinstance(step, RunStep):
678
- commands.append(step.command)
679
- elif isinstance(step, WorkdirStep):
680
- commands.append(f"cd {step.path}")
681
- content = "#!/bin/bash\n{body}".format(body="\n".join(commands))
682
- console.print(
683
- f"\n[bold]Created prepare.sh script to run before packaging ✅[/bold]"
684
- )
685
- manifest_panel = Panel(
686
- Syntax(
687
- content,
688
- "bash",
689
- theme="monokai",
690
- background_color="default",
691
- line_numbers=True,
692
- ),
693
- box=box.SQUARE,
694
- border_style="bright_black",
695
- expand=False,
696
- )
697
- console.print(manifest_panel, markup=False, highlight=True)
698
- self.prepare_bash_script.write_text(content)
699
- self.prepare_bash_script.chmod(0o755)
700
-
701
- def finalize_build(self, serve: Serve) -> None:
702
- pass
703
-
704
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
705
- sh.Command(f"{self.prepare_bash_script.absolute()}")(
706
- _out=write_stdout, _err=write_stderr
707
- )
708
-
709
- def build_serve(self, serve: Serve) -> None:
710
- # Remember serve configuration for run-time
711
- console.print("\n[bold]Building serve[/bold]")
712
- self.serve_bin_path.mkdir(parents=True, exist_ok=False)
713
- path = self.get_path() / ".path"
714
- path_text = path.read_text()
715
- console.print(f"[bold]Serve Commands:[/bold]")
716
- for command in serve.commands:
717
- console.print(f"* {command}")
718
- command_path = self.serve_bin_path / command
719
- env_vars = ""
720
- if serve.env:
721
- env_vars = " ".join([f"{k}={v}" for k, v in serve.env.items()])
722
-
723
- content = f"#!/bin/bash\ncd {serve.cwd}\nPATH={path_text}:$PATH {env_vars} {serve.commands[command]}"
724
- command_path.write_text(content)
725
- manifest_panel = Panel(
726
- Syntax(
727
- content.strip(),
728
- "bash",
729
- theme="monokai",
730
- background_color="default",
731
- line_numbers=True,
732
- ),
733
- box=box.SQUARE,
734
- border_style="bright_black",
735
- expand=False,
736
- )
737
- console.print(manifest_panel, markup=False, highlight=True)
738
- command_path.chmod(0o755)
739
-
740
- def run_serve_command(self, command: str) -> None:
741
- console.print(f"\n[bold]Running {command} command[/bold]")
742
- command_path = self.serve_bin_path / command
743
- sh.Command(str(command_path))(_out=write_stdout, _err=write_stderr)
744
-
745
-
746
- class WasmerBuilder:
747
- def get_build_mount_path(self, name: str) -> Path:
748
- return self.inner_builder.get_build_mount_path(name)
749
-
750
- def get_serve_mount_path(self, name: str) -> Path:
751
- if name == "app":
752
- return Path("/app")
753
- else:
754
- return Path("/opt") / name
755
-
756
- mapper: Dict[str, MapperItem] = {
757
- "python": {
758
- "dependencies": {
759
- "latest": "python/python@=3.13.1",
760
- "3.13": "python/python@=3.13.1",
761
- },
762
- "scripts": {"python"},
763
- "aliases": {},
764
- "env": {
765
- "PYTHONEXECUTABLE": "/bin/python",
766
- "PYTHONHOME": "/cpython",
767
- },
768
- },
769
- "pandoc": {
770
- "dependencies": {
771
- "latest": "wasmer/pandoc@=0.0.1",
772
- "3.5": "wasmer/pandoc@=0.0.1",
773
- },
774
- "scripts": {"pandoc"},
775
- },
776
- "ffmpeg": {
777
- "dependencies": {
778
- "latest": "wasmer/ffmpeg@=1.0.5",
779
- "N-111519": "wasmer/ffmpeg@=1.0.5",
780
- },
781
- "scripts": {"ffmpeg"},
782
- },
783
- "php": {
784
- "dependencies": {
785
- "latest": "php/php-32@=8.3.2102",
786
- "8.3": "php/php-32@=8.3.2102",
787
- "8.2": "php/php-32@=8.2.2801",
788
- "8.1": "php/php-32@=8.1.3201",
789
- "7.4": "php/php-32@=7.4.3301",
790
- },
791
- "architecture_dependencies": {
792
- "64-bit": {
793
- "latest": "php/php-64@=8.3.2102",
794
- "8.3": "php/php-64@=8.3.2102",
795
- "8.2": "php/php-64@=8.2.2801",
796
- "8.1": "php/php-64@=8.1.3201",
797
- "7.4": "php/php-64@=7.4.3301",
798
- },
799
- "32-bit": {
800
- "latest": "php/php-32@=8.3.2102",
801
- "8.3": "php/php-32@=8.3.2102",
802
- "8.2": "php/php-32@=8.2.2801",
803
- "8.1": "php/php-32@=8.1.3201",
804
- "7.4": "php/php-32@=7.4.3301",
805
- },
806
- },
807
- "scripts": {"php"},
808
- "aliases": {},
809
- "env": {},
810
- },
811
- "bash": {
812
- "dependencies": {
813
- "latest": "wasmer/bash@=1.0.24",
814
- "8.3": "wasmer/bash@=1.0.24",
815
- },
816
- "scripts": {"bash", "sh"},
817
- "aliases": {},
818
- "env": {},
819
- },
820
- "static-web-server": {
821
- "dependencies": {
822
- "latest": "wasmer/static-web-server@=1.1.0",
823
- "2.38.0": "wasmer/static-web-server@=1.1.0",
824
- "0.1": "wasmer/static-web-server@=1.1.0",
825
- },
826
- "scripts": {"webserver"},
827
- "aliases": {"static-web-server": "webserver"},
828
- "env": {},
829
- },
830
- }
831
-
832
- def __init__(
833
- self,
834
- inner_builder: Builder,
835
- src_dir: Path,
836
- registry: Optional[str] = None,
837
- token: Optional[str] = None,
838
- bin: Optional[str] = None,
839
- ) -> None:
840
- self.src_dir = src_dir
841
- self.inner_builder = inner_builder
842
- # The path where we store the directory of the wasmer app in the inner builder
843
- self.wasmer_dir_path = self.src_dir / ".shipit" / "wasmer"
844
- self.wasmer_registry = registry
845
- self.wasmer_token = token
846
- self.bin = bin or "wasmer"
847
- self.default_env = {
848
- "SHIPIT_PYTHON_EXTRA_INDEX_URL": "https://pythonindex.wasix.org/simple",
849
- "SHIPIT_PYTHON_CROSS_PLATFORM": "wasix_wasm32",
850
- "SHIPIT_PYTHON_PRECOMPILE": "true",
851
- }
852
-
853
- def getenv(self, name: str) -> Optional[str]:
854
- return self.inner_builder.getenv(name) or self.default_env.get(name)
855
-
856
- def build(
857
- self, env: Dict[str, str], mounts: List[Mount], build: List[Step]
858
- ) -> None:
859
- return self.inner_builder.build(env, mounts, build)
860
-
861
- def build_prepare(self, serve: Serve) -> None:
862
- print("Building prepare")
863
- prepare_dir = self.wasmer_dir_path / "prepare"
864
- prepare_dir.mkdir(parents=True, exist_ok=True)
865
- env = serve.env or {}
866
- for dep in serve.deps:
867
- if dep.name in self.mapper:
868
- dep_env = self.mapper[dep.name].get("env")
869
- if dep_env is not None:
870
- env.update(dep_env)
871
- if env:
872
- env_lines = [f"export {k}={v}" for k, v in env.items()]
873
- env_lines = "\n".join(env_lines)
874
- else:
875
- env_lines = ""
876
-
877
- commands: List[str] = []
878
- if serve.cwd:
879
- commands.append(f"cd {serve.cwd}")
880
-
881
- if serve.prepare:
882
- for step in serve.prepare:
883
- if isinstance(step, RunStep):
884
- commands.append(step.command)
885
- elif isinstance(step, WorkdirStep):
886
- commands.append(f"cd {step.path}")
887
-
888
- body = "\n".join(filter(None, [env_lines, *commands]))
889
- content = f"#!/bin/bash\n\n{body}"
890
- console.print(
891
- f"\n[bold]Created prepare.sh script to run before packaging ✅[/bold]"
892
- )
893
- manifest_panel = Panel(
894
- Syntax(
895
- content,
896
- "bash",
897
- theme="monokai",
898
- background_color="default",
899
- line_numbers=True,
900
- ),
901
- box=box.SQUARE,
902
- border_style="bright_black",
903
- expand=False,
904
- )
905
- console.print(manifest_panel, markup=False, highlight=True)
906
-
907
- (prepare_dir / "prepare.sh").write_text(
908
- content,
909
- )
910
- (prepare_dir / "prepare.sh").chmod(0o755)
911
-
912
- def finalize_build(self, serve: Serve) -> None:
913
- inner = cast(Any, self.inner_builder)
914
- inner.finalize_build(serve)
915
-
916
- def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
917
- prepare_dir = self.wasmer_dir_path / "prepare"
918
- self.run_serve_command(
919
- "bash",
920
- extra_args=[
921
- f"--mapdir=/prepare:{prepare_dir}",
922
- "--",
923
- "/prepare/prepare.sh",
924
- ],
925
- )
926
-
927
- def build_serve(self, serve: Serve) -> None:
928
- from tomlkit import comment, document, nl, table, aot, string, array
929
-
930
- doc = document()
931
- doc.add(comment(f"Wasmer manifest generated with Shipit v{shipit_version}"))
932
- package = table()
933
- doc.add("package", package)
934
- package.add("entrypoint", "start")
935
- dependencies = table()
936
- doc.add("dependencies", dependencies)
937
-
938
- binaries = {}
939
-
940
- deps = serve.deps or []
941
- # We add bash if it's not present, as the prepare command is run in bash
942
- if serve.prepare:
943
- if not any(dep.name == "bash" for dep in deps):
944
- deps.append(Package("bash"))
945
-
946
- if deps:
947
- console.print(f"[bold]Mapping dependencies to Wasmer packages:[/bold]")
948
- for dep in deps:
949
- if dep.name in self.mapper:
950
- version = dep.version or "latest"
951
- mapped_dependencies = self.mapper[dep.name]["dependencies"]
952
- if dep.architecture:
953
- architecture_dependencies = (
954
- self.mapper[dep.name]
955
- .get("architecture_dependencies", {})
956
- .get(dep.architecture, {})
957
- )
958
- if architecture_dependencies:
959
- mapped_dependencies = architecture_dependencies
960
- if version in mapped_dependencies:
961
- console.print(
962
- f"* {dep.name}@{version} mapped to {self.mapper[dep.name]['dependencies'][version]}"
963
- )
964
- package_name, version = mapped_dependencies[version].split("@")
965
- dependencies.add(package_name, version)
966
- scripts = self.mapper[dep.name].get("scripts") or []
967
- for script in scripts:
968
- binaries[script] = {
969
- "script": f"{package_name}:{script}",
970
- "env": self.mapper[dep.name].get("env"),
971
- }
972
- aliases = self.mapper[dep.name].get("aliases") or {}
973
- for alias, script in aliases.items():
974
- binaries[alias] = {
975
- "script": f"{package_name}:{script}",
976
- "env": self.mapper[dep.name].get("env"),
977
- }
978
- else:
979
- raise Exception(
980
- f"Dependency {dep.name}@{version} not found in Wasmer"
981
- )
982
- else:
983
- raise Exception(f"Dependency {dep.name} not found in Wasmer")
984
-
985
- fs = table()
986
- doc.add("fs", fs)
987
- inner = cast(Any, self.inner_builder)
988
- if serve.mounts:
989
- for mount in serve.mounts:
990
- fs.add(
991
- str(mount.serve_path.absolute()),
992
- str(self.inner_builder.get_serve_mount_path(mount.name).absolute()),
993
- )
994
-
995
- doc.add(nl())
996
- if serve.commands:
997
- commands = aot()
998
- doc.add("command", commands)
999
- for command_name, command_line in serve.commands.items():
1000
- command = table()
1001
- commands.append(command)
1002
- parts = shlex.split(command_line)
1003
- program = parts[0]
1004
- command.add("name", command_name)
1005
- program_binary = binaries[program]
1006
- command.add("module", program_binary["script"])
1007
- command.add("runner", "wasi")
1008
- wasi_args = table()
1009
- if serve.cwd:
1010
- wasi_args.add("cwd", serve.cwd)
1011
- wasi_args.add("main-args", array(parts[1:]).multiline(True))
1012
- env = program_binary.get("env") or {}
1013
- if serve.env:
1014
- env.update(serve.env)
1015
- if env:
1016
- arr = array([f"{k}={v}" for k, v in env.items()]).multiline(True)
1017
- wasi_args.add("env", arr)
1018
- title = string("annotations.wasi", literal=False)
1019
- command.add(title, wasi_args)
1020
-
1021
- inner = cast(Any, self.inner_builder)
1022
- self.wasmer_dir_path.mkdir(parents=True, exist_ok=True)
1023
-
1024
- manifest = doc.as_string().replace(
1025
- '[command."annotations.wasi"]', "[command.annotations.wasi]"
1026
- )
1027
- console.print(f"\n[bold]Created wasmer.toml manifest ✅[/bold]")
1028
- manifest_panel = Panel(
1029
- Syntax(
1030
- manifest.strip(),
1031
- "toml",
1032
- theme="monokai",
1033
- background_color="default",
1034
- line_numbers=True,
1035
- ),
1036
- box=box.SQUARE,
1037
- border_style="bright_black",
1038
- expand=False,
1039
- )
1040
- console.print(manifest_panel, markup=False, highlight=True)
1041
- (self.wasmer_dir_path / "wasmer.toml").write_text(manifest)
1042
-
1043
- original_app_yaml_path = self.src_dir / "app.yaml"
1044
- if original_app_yaml_path.exists():
1045
- console.print(
1046
- f"[bold]Using original app.yaml found in source directory[/bold]"
1047
- )
1048
- yaml_config = yaml.safe_load(original_app_yaml_path.read_text())
1049
- else:
1050
- yaml_config = {
1051
- "kind": "wasmer.io/App.v0",
1052
- }
1053
- # Update the app to use the new package
1054
- yaml_config["package"] = "."
1055
- if serve.services:
1056
- capabilities = yaml_config.get("capabilities", {})
1057
- has_mysql = any(service.provider == "mysql" for service in serve.services)
1058
- # has_postgres = any(service.provider == "postgres" for service in serve.services)
1059
- # has_redis = any(service.provider == "redis" for service in serve.services)
1060
- if has_mysql:
1061
- capabilities["database"] = {"engine": "mysql"}
1062
- yaml_config["capabilities"] = capabilities
1063
-
1064
- # Attach declared volumes to the app manifest (serve-time mounts)
1065
- if serve.volumes:
1066
- volumes_yaml = yaml_config.get("volumes", [])
1067
- for vol in serve.volumes:
1068
- volumes_yaml.append(
1069
- {
1070
- "name": vol.name,
1071
- "mount": str(vol.serve_path),
1072
- }
1073
- )
1074
- yaml_config["volumes"] = volumes_yaml
1075
-
1076
- # If it has a php dependency, set the scaling mode to single_concurrency
1077
- has_php = any(dep.name == "php" for dep in serve.deps)
1078
- if has_php:
1079
- scaling = yaml_config.get("scaling", {})
1080
- scaling["mode"] = "single_concurrency"
1081
- yaml_config["scaling"] = scaling
1082
-
1083
- if "after_deploy" in serve.commands:
1084
- jobs = yaml_config.get("jobs", [])
1085
- jobs.append(
1086
- {
1087
- "name": "after_deploy",
1088
- "trigger": "post-deployment",
1089
- "action": {"execute": {"command": "after_deploy"}},
1090
- }
1091
- )
1092
- yaml_config["jobs"] = jobs
1093
-
1094
- app_yaml = yaml.dump(
1095
- yaml_config,
1096
- )
1097
-
1098
- console.print(f"\n[bold]Created app.yaml manifest ✅[/bold]")
1099
- app_yaml_panel = Panel(
1100
- Syntax(
1101
- app_yaml.strip(),
1102
- "yaml",
1103
- theme="monokai",
1104
- background_color="default",
1105
- line_numbers=True,
1106
- ),
1107
- box=box.SQUARE,
1108
- border_style="bright_black",
1109
- expand=False,
1110
- )
1111
- console.print(app_yaml_panel, markup=False, highlight=True)
1112
- (self.wasmer_dir_path / "app.yaml").write_text(app_yaml)
1113
-
1114
- # self.inner_builder.build_serve(serve)
1115
-
1116
- def run_serve_command(
1117
- self, command: str, extra_args: Optional[List[str]] = None
1118
- ) -> None:
1119
- console.print(f"\n[bold]Serving site[/bold]: running {command} command")
1120
- extra_args = extra_args or []
1121
-
1122
- if self.wasmer_registry:
1123
- extra_args = [f"--registry={self.wasmer_registry}"] + extra_args
1124
- self.run_command(
1125
- self.bin,
1126
- [
1127
- "run",
1128
- str(self.wasmer_dir_path.absolute()),
1129
- "--net",
1130
- f"--command={command}",
1131
- *extra_args,
1132
- ],
1133
- )
1134
-
1135
- def run_command(
1136
- self, command: str, extra_args: Optional[List[str]] | None = None
1137
- ) -> Any:
1138
- sh.Command(command)(
1139
- *(extra_args or []), _out=write_stdout, _err=write_stderr, _env=os.environ
1140
- )
1141
-
1142
- def deploy_config(self, config_path: Path) -> None:
1143
- package_webc_path = self.wasmer_dir_path / "package.webc"
1144
- app_yaml_path = self.wasmer_dir_path / "app.yaml"
1145
- package_webc_path.parent.mkdir(parents=True, exist_ok=True)
1146
- self.run_command(
1147
- self.bin,
1148
- ["package", "build", self.wasmer_dir_path, "--out", package_webc_path],
1149
- )
1150
- config_path.write_text(
1151
- json.dumps(
1152
- {
1153
- "app_yaml_path": str(app_yaml_path.absolute()),
1154
- "package_webc_path": str(package_webc_path.absolute()),
1155
- "package_webc_size": package_webc_path.stat().st_size,
1156
- "package_webc_sha256": hashlib.sha256(
1157
- package_webc_path.read_bytes()
1158
- ).hexdigest(),
1159
- }
1160
- )
1161
- )
1162
- console.print(f"\n[bold]Saved deploy config to {config_path}[/bold]")
1163
-
1164
- def deploy(
1165
- self, app_owner: Optional[str] = None, app_name: Optional[str] = None
1166
- ) -> str:
1167
- extra_args = []
1168
- if self.wasmer_registry:
1169
- extra_args += ["--registry", self.wasmer_registry]
1170
- if self.wasmer_token:
1171
- extra_args += ["--token", self.wasmer_token]
1172
- if app_owner:
1173
- extra_args += ["--owner", app_owner]
1174
- if app_name:
1175
- extra_args += ["--app-name", app_name]
1176
- # self.run_command(
1177
- # self.bin,
1178
- # [
1179
- # "package",
1180
- # "push",
1181
- # self.wasmer_dir_path,
1182
- # "--namespace",
1183
- # app_owner,
1184
- # "--non-interactive",
1185
- # *extra_args,
1186
- # ],
1187
- # )
1188
- return self.run_command(
1189
- self.bin,
1190
- [
1191
- "deploy",
1192
- "--publish-package",
1193
- "--dir",
1194
- self.wasmer_dir_path,
1195
- "--non-interactive",
1196
- *extra_args,
1197
- ],
1198
- )
49
+ serve_path: str
1199
50
 
1200
51
 
1201
52
  class Ctx:
1202
- def __init__(self, builder: Builder) -> None:
1203
- self.builder = builder
53
+ def __init__(self, build_backend: BuildBackend, runner: Runner) -> None:
54
+ self.build_backend = build_backend
55
+ self.runner = runner
1204
56
  self.packages: Dict[str, Package] = {}
1205
57
  self.builds: List[Build] = []
1206
58
  self.steps: List[Step] = []
@@ -1208,7 +60,6 @@ class Ctx:
1208
60
  self.mounts: List[Mount] = []
1209
61
  self.volumes: List[Volume] = []
1210
62
  self.services: Dict[str, Service] = {}
1211
- self.getenv_variables: Set[str] = set()
1212
63
 
1213
64
  def add_package(self, package: Package) -> str:
1214
65
  index = f"{package.name}@{package.version}" if package.version else package.name
@@ -1254,10 +105,6 @@ class Ctx:
1254
105
  self.steps.append(step)
1255
106
  return f"ref:step:{len(self.steps) - 1}"
1256
107
 
1257
- def getenv(self, name: str) -> Optional[str]:
1258
- self.getenv_variables.add(name)
1259
- return self.builder.getenv(name)
1260
-
1261
108
  def dep(
1262
109
  self,
1263
110
  name: str,
@@ -1306,7 +153,7 @@ class Ctx:
1306
153
  commands=commands,
1307
154
  prepare=prepare_steps,
1308
155
  workers=workers,
1309
- mounts=self.get_refs([mount["ref"] for mount in mounts])
156
+ mounts=self.get_refs([mount.ref for mount in mounts])
1310
157
  if mounts
1311
158
  else None,
1312
159
  volumes=self.get_refs([volume["ref"] for volume in volumes])
@@ -1355,15 +202,12 @@ class Ctx:
1355
202
  return f"ref:mount:{len(self.mounts) - 1}"
1356
203
 
1357
204
  def mount(self, name: str) -> Optional[str]:
1358
- build_path = self.builder.get_build_mount_path(name)
1359
- serve_path = self.builder.get_serve_mount_path(name)
205
+ build_path = self.build_backend.get_build_mount_path(name)
206
+ serve_path = self.runner.get_serve_mount_path(name)
1360
207
  mount = Mount(name, build_path, serve_path)
1361
208
  ref = self.add_mount(mount)
1362
- return {
1363
- "ref": ref,
1364
- "build": str(build_path.absolute()),
1365
- "serve": str(serve_path.absolute()),
1366
- }
209
+
210
+ return CtxMount(ref=ref, path=str(build_path.absolute()), serve_path=str(serve_path.absolute()))
1367
211
 
1368
212
  def add_volume(self, volume: Volume) -> Optional[str]:
1369
213
  self.volumes.append(volume)
@@ -1379,35 +223,76 @@ class Ctx:
1379
223
  }
1380
224
 
1381
225
 
1382
- def evaluate_shipit(shipit_file: Path, builder: Builder) -> Tuple[Ctx, Serve]:
226
+ def evaluate_shipit(
227
+ shipit_file: Path,
228
+ build_backend: BuildBackend,
229
+ runner: Runner,
230
+ provider_config: Config,
231
+ ) -> Tuple[Ctx, Serve]:
1383
232
  source = shipit_file.read_text()
1384
- ctx = Ctx(builder)
1385
- glb = sl.Globals.standard()
1386
- mod = sl.Module()
1387
-
1388
- mod.add_callable("service", ctx.service)
1389
- mod.add_callable("getenv", ctx.getenv)
1390
- mod.add_callable("dep", ctx.dep)
1391
- mod.add_callable("serve", ctx.serve)
1392
- mod.add_callable("run", ctx.run)
1393
- mod.add_callable("mount", ctx.mount)
1394
- mod.add_callable("volume", ctx.volume)
1395
- mod.add_callable("workdir", ctx.workdir)
1396
- mod.add_callable("copy", ctx.copy)
1397
- mod.add_callable("path", ctx.path)
1398
- mod.add_callable("env", ctx.env)
1399
- mod.add_callable("use", ctx.use)
1400
-
1401
- dialect = sl.Dialect.extended()
1402
- dialect.enable_f_strings = True
1403
-
1404
- ast = sl.parse("shipit", source, dialect=dialect)
1405
-
1406
- sl.eval(mod, ast, glb)
233
+ ctx = Ctx(build_backend, runner)
234
+ glb = sl.GlobalsBuilder.standard()
235
+
236
+ glb.set("PORT", str(provider_config.port or "8080"))
237
+ glb.set("config", provider_config)
238
+ glb.set("service", ctx.service)
239
+ glb.set("dep", ctx.dep)
240
+ glb.set("serve", ctx.serve)
241
+ glb.set("run", ctx.run)
242
+ glb.set("mount", ctx.mount)
243
+ glb.set("volume", ctx.volume)
244
+ glb.set("workdir", ctx.workdir)
245
+ glb.set("copy", ctx.copy)
246
+ glb.set("path", ctx.path)
247
+ glb.set("env", ctx.env)
248
+ glb.set("use", ctx.use)
249
+
250
+ dialect = sl.Dialect(enable_keyword_only_arguments=True, enable_f_strings=True)
251
+
252
+ ast = sl.AstModule.parse("Shipit", source, dialect=dialect)
253
+
254
+ evaluator = sl.Evaluator()
255
+ evaluator.eval_module(ast, glb.build())
1407
256
  if not ctx.serves:
1408
257
  raise ValueError(f"No serve definition found in {shipit_file}")
1409
258
  assert len(ctx.serves) <= 1, "Only one serve is allowed for now"
1410
259
  serve = next(iter(ctx.serves.values()))
260
+
261
+ # Now we apply the custom commands (start, after_deploy, build, install)
262
+ if provider_config.commands.start:
263
+ serve.commands["start"] = provider_config.commands.start
264
+
265
+ if provider_config.commands.after_deploy:
266
+ serve.commands["after_deploy"] = provider_config.commands.after_deploy
267
+
268
+ if provider_config.commands.build or provider_config.commands.install:
269
+ new_build = []
270
+ has_done_build = False
271
+ has_done_install = False
272
+ for step in serve.build:
273
+ if isinstance(step, RunStep):
274
+ if step.group == "build" and not has_done_build and provider_config.commands.build:
275
+ new_build.append(RunStep(provider_config.commands.build, group="build"))
276
+ has_done_build = True
277
+ elif step.group == "install" and not has_done_install and provider_config.commands.install:
278
+ new_build.append(RunStep(provider_config.commands.install, group="install"))
279
+ has_done_install = True
280
+ else:
281
+ new_build.append(step)
282
+ else:
283
+ new_build.append(step)
284
+ if not has_done_install and provider_config.commands.install:
285
+ new_build.append(RunStep(provider_config.commands.install, group="install"))
286
+ if not has_done_build and provider_config.commands.build:
287
+ new_build.append(RunStep(provider_config.commands.build, group="build"))
288
+ serve.build = new_build
289
+
290
+ if serve.commands.get("start"):
291
+ serve.commands["start"] = serve.commands["start"].replace("$PORT", str(provider_config.port or "8080"))
292
+
293
+ if serve.commands.get("after_deploy"):
294
+ serve.commands["after_deploy"] = serve.commands["after_deploy"].replace("$PORT", str(provider_config.port or "8080"))
295
+
1411
296
  return ctx, serve
1412
297
 
1413
298
 
@@ -1421,13 +306,6 @@ def print_help() -> None:
1421
306
  console.print(panel)
1422
307
 
1423
308
 
1424
- def download_file(url: str, path: Path) -> None:
1425
- response = requests.get(url)
1426
- response.raise_for_status()
1427
- path.parent.mkdir(parents=True, exist_ok=True)
1428
- path.write_bytes(response.content)
1429
-
1430
-
1431
309
  @app.command(name="auto")
1432
310
  def auto(
1433
311
  path: Path = typer.Argument(
@@ -1499,10 +377,6 @@ def auto(
1499
377
  None,
1500
378
  help="Name of the Wasmer app.",
1501
379
  ),
1502
- use_procfile: bool = typer.Option(
1503
- True,
1504
- help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
1505
- ),
1506
380
  install_command: Optional[str] = typer.Option(
1507
381
  None,
1508
382
  help="The install command to use (overwrites the default)",
@@ -1519,10 +393,18 @@ def auto(
1519
393
  None,
1520
394
  help="The environment to use (defaults to `.env`, it will use .env.<env_name> if provided)",
1521
395
  ),
1522
- use_provider: Optional[str] = typer.Option(
396
+ provider: Optional[str] = typer.Option(
1523
397
  None,
1524
398
  help="Use a specific provider to build the project.",
1525
399
  ),
400
+ config: Optional[str] = typer.Option(
401
+ None,
402
+ help="The JSON content to use as input.",
403
+ ),
404
+ serve_port: Optional[int] = typer.Option(
405
+ None,
406
+ help="The port to use (defaults to 8080).",
407
+ ),
1526
408
  ):
1527
409
  if not path.exists():
1528
410
  raise Exception(f"The path {path} does not exist")
@@ -1545,16 +427,19 @@ def auto(
1545
427
  generate(
1546
428
  path,
1547
429
  out=shipit_path,
1548
- use_procfile=use_procfile,
1549
430
  install_command=install_command,
1550
431
  build_command=build_command,
1551
432
  start_command=start_command,
1552
- use_provider=use_provider,
433
+ provider=provider,
434
+ config=config,
1553
435
  )
1554
436
 
1555
437
  build(
1556
438
  path,
1557
439
  shipit_path=shipit_path,
440
+ install_command=install_command,
441
+ build_command=build_command,
442
+ start_command=start_command,
1558
443
  wasmer=(wasmer or wasmer_deploy),
1559
444
  docker=docker,
1560
445
  docker_client=docker_client,
@@ -1564,6 +449,9 @@ def auto(
1564
449
  wasmer_bin=wasmer_bin,
1565
450
  skip_prepare=skip_prepare,
1566
451
  env_name=env_name,
452
+ serve_port=serve_port,
453
+ provider=provider,
454
+ config=config,
1567
455
  )
1568
456
  if start or wasmer_deploy or wasmer_deploy_config:
1569
457
  serve(
@@ -1598,10 +486,6 @@ def generate(
1598
486
  "--shipit-path",
1599
487
  help="Output path (defaults to the Shipit file in the provided path).",
1600
488
  ),
1601
- use_procfile: bool = typer.Option(
1602
- True,
1603
- help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
1604
- ),
1605
489
  install_command: Optional[str] = typer.Option(
1606
490
  None,
1607
491
  help="The install command to use (overwrites the default)",
@@ -1614,41 +498,48 @@ def generate(
1614
498
  None,
1615
499
  help="The start command to use (overwrites the default)",
1616
500
  ),
1617
- use_provider: Optional[str] = typer.Option(
501
+ provider: Optional[str] = typer.Option(
1618
502
  None,
1619
503
  help="Use a specific provider to build the project.",
1620
504
  ),
505
+ config: Optional[str] = typer.Option(
506
+ None,
507
+ help="The JSON content to use as input.",
508
+ ),
1621
509
  ):
1622
510
  if not path.exists():
1623
511
  raise Exception(f"The path {path} does not exist")
1624
512
 
1625
513
  if out is None:
1626
514
  out = path / "Shipit"
1627
- custom_commands = CustomCommands()
1628
- # if (path / "Dockerfile").exists():
1629
- # # We get the start command from the Dockerfile
1630
- # with open(path / "Dockerfile", "r") as f:
1631
- # cmd = None
1632
- # for line in f:
1633
- # if line.startswith("CMD "):
1634
- # cmd = line[4:].strip()
1635
- # cmd = json.loads(cmd)
1636
- # # We get the last command
1637
- # if cmd:
1638
- # if isinstance(cmd, list):
1639
- # cmd = " ".join(cmd)
1640
- # custom_commands.start = cmd
1641
- if use_procfile:
1642
- if (path / "Procfile").exists():
1643
- procfile = Procfile.loads((path / "Procfile").read_text())
1644
- custom_commands.start = procfile.get_start_command()
515
+
516
+ base_config = Config()
517
+ base_config.commands.enrich_from_path(path)
1645
518
  if start_command:
1646
- custom_commands.start = start_command
519
+ base_config.commands.start = start_command
1647
520
  if install_command:
1648
- custom_commands.install = install_command
521
+ base_config.commands.install = install_command
1649
522
  if build_command:
1650
- custom_commands.build = build_command
1651
- content = generate_shipit(path, custom_commands, use_provider=use_provider)
523
+ base_config.commands.build = build_command
524
+ provider_cls = load_provider(path, base_config, use_provider=provider)
525
+ provider_config = load_provider_config(provider_cls, path, base_config, config=config)
526
+ provider = provider_cls(path, provider_config)
527
+ content = generate_shipit(path, provider)
528
+ config_json = provider_config.model_dump_json(indent=2, exclude_defaults=True)
529
+ if config_json and config_json != "{}":
530
+ manifest_panel = Panel(
531
+ Syntax(
532
+ config_json,
533
+ "json",
534
+ theme="monokai",
535
+ background_color="default",
536
+ line_numbers=True,
537
+ ),
538
+ box=box.SQUARE,
539
+ border_style="bright_black",
540
+ expand=False,
541
+ )
542
+ console.print(manifest_panel, markup=False, highlight=True)
1652
543
  out.write_text(content)
1653
544
  console.print(f"[bold]Generated Shipit[/bold] at {out.absolute()}")
1654
545
 
@@ -1728,22 +619,35 @@ def serve(
1728
619
  if not path.exists():
1729
620
  raise Exception(f"The path {path} does not exist")
1730
621
 
1731
- builder: Builder
1732
622
  if docker or docker_client:
1733
- builder = DockerBuilder(path, docker_client)
623
+ build_backend: BuildBackend = DockerBuildBackend(
624
+ path, ASSETS_PATH, docker_client
625
+ )
1734
626
  else:
1735
- builder = LocalBuilder(path)
1736
- if wasmer or wasmer_deploy or wasmer_deploy_config:
1737
- builder = WasmerBuilder(
1738
- builder, path, registry=wasmer_registry, token=wasmer_token, bin=wasmer_bin
627
+ build_backend = LocalBuildBackend(path, ASSETS_PATH)
628
+
629
+ needs_wasmer = wasmer or wasmer_deploy or wasmer_deploy_config
630
+ if needs_wasmer:
631
+ runner: Runner = WasmerRunner(
632
+ build_backend,
633
+ path,
634
+ registry=wasmer_registry,
635
+ token=wasmer_token,
636
+ bin=wasmer_bin,
1739
637
  )
638
+ else:
639
+ runner = LocalRunner(build_backend, path)
1740
640
 
1741
641
  if wasmer_deploy_config:
1742
- builder.deploy_config(wasmer_deploy_config)
642
+ if not isinstance(runner, WasmerRunner):
643
+ raise RuntimeError("--wasmer-deploy-config requires the Wasmer runner")
644
+ runner.deploy_config(wasmer_deploy_config)
1743
645
  elif wasmer_deploy:
1744
- builder.deploy(app_owner=wasmer_app_owner, app_name=wasmer_app_name)
646
+ if not isinstance(runner, WasmerRunner):
647
+ raise RuntimeError("--wasmer-deploy requires the Wasmer runner")
648
+ runner.deploy(app_owner=wasmer_app_owner, app_name=wasmer_app_name)
1745
649
  elif start:
1746
- builder.run_serve_command("start")
650
+ runner.run_serve_command("start")
1747
651
 
1748
652
 
1749
653
  @app.command(name="plan")
@@ -1796,10 +700,6 @@ def plan(
1796
700
  None,
1797
701
  help="Use a specific Docker client (such as depot, podman, etc.)",
1798
702
  ),
1799
- use_procfile: bool = typer.Option(
1800
- True,
1801
- help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
1802
- ),
1803
703
  install_command: Optional[str] = typer.Option(
1804
704
  None,
1805
705
  help="The install command to use (overwrites the default)",
@@ -1812,10 +712,18 @@ def plan(
1812
712
  None,
1813
713
  help="The start command to use (overwrites the default)",
1814
714
  ),
1815
- use_provider: Optional[str] = typer.Option(
715
+ provider: Optional[str] = typer.Option(
1816
716
  None,
1817
717
  help="Use a specific provider to build the project.",
1818
718
  ),
719
+ config: Optional[str] = typer.Option(
720
+ None,
721
+ help="The JSON content to use as input.",
722
+ ),
723
+ serve_port: Optional[int] = typer.Option(
724
+ None,
725
+ help="The port to use (defaults to 8080).",
726
+ ),
1819
727
  ) -> None:
1820
728
  if not path.exists():
1821
729
  raise Exception(f"The path {path} does not exist")
@@ -1838,39 +746,46 @@ def plan(
1838
746
  generate(
1839
747
  path,
1840
748
  out=shipit_path,
1841
- use_procfile=use_procfile,
1842
749
  install_command=install_command,
1843
750
  build_command=build_command,
1844
751
  start_command=start_command,
1845
- use_provider=use_provider,
752
+ provider=provider,
753
+ config=config,
1846
754
  )
1847
755
 
1848
- custom_commands = CustomCommands()
1849
- procfile_path = path / "Procfile"
1850
- if procfile_path.exists():
1851
- try:
1852
- procfile = Procfile.loads(procfile_path.read_text())
1853
- custom_commands.start = procfile.get_start_command()
1854
- except Exception:
1855
- pass
1856
-
1857
756
  shipit_file = get_shipit_path(path, shipit_path)
1858
757
 
1859
- builder: Builder
1860
758
  if docker or docker_client:
1861
- builder = DockerBuilder(path, docker_client)
759
+ build_backend: BuildBackend = DockerBuildBackend(
760
+ path, ASSETS_PATH, docker_client
761
+ )
1862
762
  else:
1863
- builder = LocalBuilder(path)
763
+ build_backend = LocalBuildBackend(path, ASSETS_PATH)
1864
764
  if wasmer:
1865
- builder = WasmerBuilder(
1866
- builder, path, registry=wasmer_registry, token=wasmer_token, bin=wasmer_bin
765
+ runner: Runner = WasmerRunner(
766
+ build_backend,
767
+ path,
768
+ registry=wasmer_registry,
769
+ token=wasmer_token,
770
+ bin=wasmer_bin,
1867
771
  )
772
+ else:
773
+ runner = LocalRunner(build_backend, path)
1868
774
 
1869
- ctx, serve = evaluate_shipit(shipit_file, builder)
1870
- metadata_commands: Dict[str, Optional[str]] = {
1871
- "start": serve.commands.get("start"),
1872
- "after_deploy": serve.commands.get("after_deploy"),
1873
- }
775
+ base_config = Config()
776
+ base_config.commands.enrich_from_path(path)
777
+ if install_command:
778
+ base_config.commands.install = install_command
779
+ if build_command:
780
+ base_config.commands.build = build_command
781
+ if start_command:
782
+ base_config.commands.start = start_command
783
+ if serve_port:
784
+ base_config.port = serve_port
785
+ provider_cls = load_provider(path, base_config, use_provider=provider)
786
+ provider_config = load_provider_config(provider_cls, path, base_config, config=config)
787
+ # provider_config = runner.prepare_config(provider_config)
788
+ ctx, serve = evaluate_shipit(shipit_file, build_backend, runner, provider_config)
1874
789
 
1875
790
  def _collect_group_commands(group: str) -> Optional[str]:
1876
791
  commands = [
@@ -1882,25 +797,23 @@ def plan(
1882
797
  return None
1883
798
  return " && ".join(commands)
1884
799
 
1885
- metadata_install = _collect_group_commands("install")
1886
- metadata_build = _collect_group_commands("build")
1887
- metadata_commands["install"] = metadata_install
1888
- metadata_commands["build"] = metadata_build
1889
- platform: Optional[str]
1890
- try:
1891
- provider_cls = detect_provider(path, custom_commands)
1892
- provider_instance = provider_cls(path, custom_commands)
1893
- provider_instance.initialize()
1894
- platform = provider_instance.platform()
1895
- except Exception:
1896
- platform = None
800
+ start_command = serve.commands.get("start")
801
+ after_deploy_command = serve.commands.get("after_deploy")
802
+ install_command = _collect_group_commands("install")
803
+ build_command = _collect_group_commands("build")
804
+ if start_command:
805
+ provider_config.commands.start = start_command
806
+ if after_deploy_command:
807
+ provider_config.commands.after_deploy = after_deploy_command
808
+ if install_command:
809
+ provider_config.commands.install = install_command
810
+ if build_command:
811
+ provider_config.commands.build = build_command
1897
812
  plan_output = {
1898
- "provider": serve.provider,
1899
- "metadata": {
1900
- "platform": platform,
1901
- "commands": metadata_commands,
1902
- },
1903
- "config": sorted(ctx.getenv_variables),
813
+ "provider": provider_cls.name(),
814
+ "config": json.loads(
815
+ provider_config.model_dump_json(exclude_defaults=True)
816
+ ),
1904
817
  "services": [
1905
818
  {"name": svc.name, "provider": svc.provider}
1906
819
  for svc in (serve.services or [])
@@ -1927,6 +840,18 @@ def build(
1927
840
  None,
1928
841
  help="The path to the Shipit file (defaults to Shipit in the provided path).",
1929
842
  ),
843
+ start_command: Optional[str] = typer.Option(
844
+ None,
845
+ help="The start command to use (overwrites the default)",
846
+ ),
847
+ install_command: Optional[str] = typer.Option(
848
+ None,
849
+ help="The install command to use (overwrites the default)",
850
+ ),
851
+ build_command: Optional[str] = typer.Option(
852
+ None,
853
+ help="The build command to use (overwrites the default)",
854
+ ),
1930
855
  wasmer: bool = typer.Option(
1931
856
  False,
1932
857
  help="Use Wasmer to build and serve the project.",
@@ -1963,23 +888,59 @@ def build(
1963
888
  None,
1964
889
  help="The environment to use (defaults to `.env`, it will use .env.<env_name> if provided)",
1965
890
  ),
891
+ serve_port: Optional[int] = typer.Option(
892
+ None,
893
+ help="The port to use (defaults to 8080).",
894
+ ),
895
+ provider: Optional[str] = typer.Option(
896
+ None,
897
+ help="Use a specific provider to build the project.",
898
+ ),
899
+ config: Optional[str] = typer.Option(
900
+ None,
901
+ help="The JSON content to use as input.",
902
+ ),
1966
903
  ) -> None:
1967
904
  if not path.exists():
1968
905
  raise Exception(f"The path {path} does not exist")
1969
906
 
1970
907
  shipit_file = get_shipit_path(path, shipit_path)
1971
908
 
1972
- builder: Builder
1973
909
  if docker or docker_client:
1974
- builder = DockerBuilder(path, docker_client)
910
+ build_backend: BuildBackend = DockerBuildBackend(
911
+ path, ASSETS_PATH, docker_client
912
+ )
1975
913
  else:
1976
- builder = LocalBuilder(path)
914
+ build_backend = LocalBuildBackend(path, ASSETS_PATH)
1977
915
  if wasmer:
1978
- builder = WasmerBuilder(
1979
- builder, path, registry=wasmer_registry, token=wasmer_token, bin=wasmer_bin
916
+ runner: Runner = WasmerRunner(
917
+ build_backend,
918
+ path,
919
+ registry=wasmer_registry,
920
+ token=wasmer_token,
921
+ bin=wasmer_bin,
1980
922
  )
923
+ else:
924
+ runner = LocalRunner(build_backend, path)
1981
925
 
1982
- ctx, serve = evaluate_shipit(shipit_file, builder)
926
+ base_config = Config()
927
+ base_config.commands.enrich_from_path(path)
928
+ if start_command:
929
+ base_config.commands.start = start_command
930
+ if install_command:
931
+ base_config.commands.install = install_command
932
+ if build_command:
933
+ base_config.commands.build = build_command
934
+ serve_port = serve_port or os.environ.get("PORT")
935
+ if serve_port:
936
+ base_config.port = serve_port
937
+
938
+ provider_cls = load_provider(path, base_config, use_provider=provider)
939
+ provider_config = load_provider_config(provider_cls, path, base_config, config=config)
940
+ provider_config = runner.prepare_config(provider_config)
941
+ ctx, serve = evaluate_shipit(
942
+ shipit_file, build_backend, runner, provider_config
943
+ )
1983
944
  env = {
1984
945
  "PATH": "",
1985
946
  "COLORTERM": os.environ.get("COLORTERM", ""),
@@ -1999,6 +960,9 @@ def build(
1999
960
  return build(
2000
961
  path,
2001
962
  shipit_path=shipit_path,
963
+ install_command=install_command,
964
+ build_command=build_command,
965
+ start_command=start_command,
2002
966
  wasmer=wasmer,
2003
967
  skip_prepare=skip_prepare,
2004
968
  wasmer_bin=wasmer_bin,
@@ -2008,6 +972,9 @@ def build(
2008
972
  docker_client=None,
2009
973
  skip_docker_if_safe_build=False,
2010
974
  env_name=env_name,
975
+ serve_port=serve_port,
976
+ provider=provider,
977
+ config=config,
2011
978
  )
2012
979
 
2013
980
  serve.env = serve.env or {}
@@ -2019,14 +986,13 @@ def build(
2019
986
  env_vars = dotenv_values(path / f".env.{env_name}")
2020
987
  serve.env.update(env_vars)
2021
988
 
989
+ assert serve.commands.get("start"), "No start command could be found, please provide a start command"
990
+
2022
991
  # Build and serve
2023
- builder.build(env, serve.mounts, serve.build)
2024
- if serve.prepare:
2025
- builder.build_prepare(serve)
2026
- builder.build_serve(serve)
2027
- builder.finalize_build(serve)
992
+ build_backend.build(serve.name, env, serve.mounts or [], serve.build)
993
+ runner.build(serve)
2028
994
  if serve.prepare and not skip_prepare:
2029
- builder.prepare(env, serve.prepare)
995
+ runner.prepare(env, serve.prepare)
2030
996
 
2031
997
 
2032
998
  def get_shipit_path(path: Path, shipit_path: Optional[Path] = None) -> Path: