shipit-cli 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

shipit/cli.py

@@ -1,10 +1,12 @@
 import logging
+import requests
 import os
 import shlex
 import shutil
 import sys
 import json
 import yaml
+import base64
 from dataclasses import dataclass
 from pathlib import Path
 from typing import (
@@ -32,6 +34,8 @@ from rich.syntax import Syntax
 
 from shipit.version import version as shipit_version
 from shipit.generator import generate_shipit
+from shipit.providers.base import CustomCommands
+from shipit.procfile import Procfile
 
 
 console = Console()
@@ -49,10 +53,18 @@ class Mount:
     serve_path: Path
 
 
+@dataclass
+class Volume:
+    name: str
+    serve_path: Path
+
+
 @dataclass
 class Service:
     name: str
-    provider: Literal["postgres", "mysql", "redis"] # Right now we only support postgres and mysql
+    provider: Literal[
+        "postgres", "mysql", "redis"
+    ]  # Right now we only support postgres and mysql
 
 
 @dataclass
@@ -63,10 +75,10 @@ class Serve:
     deps: List["Package"]
     commands: Dict[str, str]
     cwd: Optional[str] = None
-    assets: Optional[Dict[str, str]] = None
     prepare: Optional[List["PrepareStep"]] = None
     workers: Optional[List[str]] = None
     mounts: Optional[List[Mount]] = None
+    volumes: Optional[List[Volume]] = None
     env: Optional[Dict[str, str]] = None
     services: Optional[List[Service]] = None
 
@@ -98,6 +110,11 @@ class CopyStep:
     source: str
     target: str
     ignore: Optional[List[str]] = None
+    # We can copy from the app source or from the shipit assets folder
+    base: Literal["source", "assets"] = "source"
+
+    def is_download(self) -> bool:
+        return self.source.startswith("http://") or self.source.startswith("https://")
 
 
 @dataclass
@@ -147,7 +164,6 @@ class Builder(Protocol):
     def build(
         self, env: Dict[str, str], mounts: List[Mount], steps: List[Step]
     ) -> None: ...
-    def build_assets(self, assets: Dict[str, str]) -> None: ...
     def build_prepare(self, serve: Serve) -> None: ...
     def build_serve(self, serve: Serve) -> None: ...
     def finalize_build(self, serve: Serve) -> None: ...
@@ -157,8 +173,6 @@ class Builder(Protocol):
     def run_command(
         self, command: str, extra_args: Optional[List[str]] | None = None
     ) -> Any: ...
-    def serve_mount(self, name: str) -> str: ...
-    def get_asset(self, name: str) -> str: ...
     def get_build_mount_path(self, name: str) -> Path: ...
     def get_serve_mount_path(self, name: str) -> Path: ...
 
@@ -258,13 +272,23 @@ class DockerBuilder:
 
     def run_command(self, command: str, extra_args: Optional[List[str]] = None) -> Any:
         image_name = self.docker_name_path.read_text()
-        return sh.Command(
-            "docker"
-        )(
+        docker_args: List[str] = [
             "run",
             "-p",
             "80:80",
             "--rm",
+        ]
+        # Attach volumes if present
+        # if serve.volumes:
+        #     for vol in serve.volumes:
+        #         docker_args += [
+        #             "--mount",
+        #             f"type=volume,source={vol.name},target={str(vol.serve_path)}",
+        #         ]
+        return sh.Command(
+            "docker"
+        )(
+            *docker_args,
             image_name,
             command,
             *(extra_args or []),
@@ -372,7 +396,28 @@ RUN curl https://mise.run | sh
                     pre = ""
                 self.docker_file_contents += f"RUN {pre}{step.command}\n"
             elif isinstance(step, CopyStep):
-                self.docker_file_contents += f"COPY {step.source} {step.target}\n"
+                if step.is_download():
+                    self.docker_file_contents += (
+                        "ADD " + step.source + " " + step.target + "\n"
+                    )
+                elif step.base == "assets":
+                    # Detect if the asset exists and is a file
+                    if (ASSETS_PATH / step.source).is_file():
+                        # Read the file content and write it to the target file
+                        content_base64 = base64.b64encode(
+                            (ASSETS_PATH / step.source).read_bytes()
+                        )
+                        self.docker_file_contents += (
+                            f"RUN echo '{content_base64}' | base64 -d > {step.target}\n"
+                        )
+                    elif (ASSETS_PATH / step.source).is_dir():
+                        raise Exception(
+                            f"Asset {step.source} is a directory, shipit doesn't currently support coppying assets directories inside Docker"
+                        )
+                    else:
+                        raise Exception(f"Asset {step.source} does not exist")
+                else:
+                    self.docker_file_contents += f"COPY {step.source} {step.target}\n"
             elif isinstance(step, EnvStep):
                 env_vars = " ".join(
                     [f"{key}={value}" for key, value in step.variables.items()]
@@ -397,9 +442,6 @@ FROM scratch
 Shipit
 """)
 
-    def build_assets(self, assets: Dict[str, str]) -> None:
-        raise NotImplementedError
-
     def get_path(self) -> Path:
         return Path("/")
 
@@ -409,11 +451,6 @@ Shipit
     def get_serve_path(self) -> Path:
         return self.get_path() / "serve"
 
-    def get_assets_path(self) -> Path:
-        path = self.get_path() / "assets"
-        self.mkdir(path)
-        return path
-
     def prepare(self, env: Dict[str, str], prepare: List[PrepareStep]) -> None:
         raise NotImplementedError
 
@@ -433,14 +470,6 @@ Shipit
                 mode=0o755,
             )
 
-    def serve_mount(self, name: str) -> str:
-        path = self.mkdir(Path("serve") / "mounts" / name)
-        return str(path.absolute())
-
-    def get_asset(self, name: str) -> str:
-        asset_path = ASSETS_PATH / name
-        return asset_path.read_text()
-
     def run_serve_command(self, command: str) -> None:
         path = self.shipit_docker_path / "serve" / "bin" / command
         self.run_command(str(path))
@@ -518,18 +547,31 @@ class LocalBuilder:
             ignore_matches = step.ignore if step.ignore else []
             ignore_matches.append(".shipit")
             ignore_matches.append("Shipit")
-            if (self.src_dir / step.source).is_dir():
-                copytree(
-                    (self.src_dir / step.source),
-                    (build_path / step.target),
-                    dirs_exist_ok=True,
-                    ignore=ignore_patterns(*ignore_matches),
-                )
+
+            if step.is_download():
+                download_file(step.source, (build_path / step.target))
             else:
-                copy(
-                    (self.src_dir / step.source),
-                    (build_path / step.target),
-                )
+                if step.base == "source":
+                    base = self.src_dir
+                elif step.base == "assets":
+                    base = ASSETS_PATH
+                else:
+                    raise Exception(f"Unknown base: {step.base}")
+
+                if (base / step.source).is_dir():
+                    copytree(
+                        (base / step.source),
+                        (build_path / step.target),
+                        dirs_exist_ok=True,
+                        ignore=ignore_patterns(*ignore_matches),
+                    )
+                elif (base / step.source).is_file():
+                    copy(
+                        (base / step.source),
+                        (build_path / step.target),
+                    )
+                else:
+                    raise Exception(f"Source {step.source} is not a file or directory")
         elif isinstance(step, EnvStep):
             print(f"Setting environment variables: {step}")
             env.update(step.variables)
@@ -591,17 +633,6 @@ class LocalBuilder:
     def get_serve_path(self) -> Path:
         return self.get_path() / "serve"
 
-    def get_assets_path(self) -> Path:
-        path = self.get_path() / "assets"
-        self.mkdir(path)
-        return path
-
-    def build_assets(self, assets: Dict[str, str]) -> None:
-        assets_path = self.get_assets_path()
-        for asset in assets:
-            asset_path = assets_path / asset
-            self.create_file(asset_path, assets[asset])
-
     def build_prepare(self, serve: Serve) -> None:
         self.prepare_bash_script.parent.mkdir(parents=True, exist_ok=True)
         commands: List[str] = []
@@ -642,6 +673,7 @@ class LocalBuilder:
         )
 
     def build_serve(self, serve: Serve) -> None:
+        # Remember serve configuration for run-time
         console.print("\n[bold]Building serve[/bold]")
         serve_command_path = self.get_serve_path() / "bin"
         serve_command_path.mkdir(parents=True, exist_ok=False)
@@ -651,10 +683,12 @@ class LocalBuilder:
         for command in serve.commands:
             console.print(f"* {command}")
             command_path = serve_command_path / command
-            content = f"#!/bin/bash\ncd {serve.cwd}\nPATH={path_text}:$PATH {serve.commands[command]}"
-            command_path.write_text(
-                content
-            )
+            env_vars = ""
+            if serve.env:
+                env_vars = " ".join([f"{k}={v}" for k, v in serve.env.items()])
+
+            content = f"#!/bin/bash\ncd {serve.cwd}\nPATH={path_text}:$PATH {env_vars} {serve.commands[command]}"
+            command_path.write_text(content)
             manifest_panel = Panel(
                 Syntax(
                     content.strip(),
@@ -676,15 +710,6 @@ class LocalBuilder:
         command_path = base_path / command
         sh.Command(str(command_path))(_out=write_stdout, _err=write_stderr)
 
-    def serve_mount(self, name: str) -> str:
-        base_path = self.get_serve_path() / "mounts" / name
-        base_path.mkdir(parents=True, exist_ok=True)
-        return str(base_path.absolute())
-
-    def get_asset(self, name: str) -> str:
-        asset_path = ASSETS_PATH / name
-        return asset_path.read_text()
-
 
 class WasmerBuilder:
     def get_build_mount_path(self, name: str) -> Path:
@@ -725,8 +750,8 @@ class WasmerBuilder:
         },
         "php": {
             "dependencies": {
-                "latest": "php/php-32@=8.3.2104",
-                "8.3": "php/php-32@=8.3.2104",
+                "latest": "php/php-32@=8.3.2102",
+                "8.3": "php/php-32@=8.3.2102",
             },
             "scripts": {"php"},
             "aliases": {},
@@ -782,9 +807,6 @@ class WasmerBuilder:
     ) -> None:
         return self.inner_builder.build(env, mounts, build)
 
-    def build_assets(self, assets: Dict[str, str]) -> None:
-        return self.inner_builder.build_assets(assets)
-
     def get_build_path(self) -> Path:
         return Path("/app")
 
@@ -908,9 +930,6 @@ class WasmerBuilder:
         fs = table()
         doc.add("fs", fs)
         inner = cast(Any, self.inner_builder)
-        if serve.assets:
-            fs.add("/assets", str((inner.get_path() / "assets").absolute()))
-        # fs.add("/app", str(inner.get_build_path().absolute()))
         if serve.mounts:
             for mount in serve.mounts:
                 fs.add(
@@ -986,22 +1005,37 @@ class WasmerBuilder:
             # has_postgres = any(service.provider == "postgres" for service in serve.services)
             # has_redis = any(service.provider == "redis" for service in serve.services)
             if has_mysql:
-                capabilities["database"] = {
-                    "engine": "mysql"
-                }
+                capabilities["database"] = {"engine": "mysql"}
             yaml_config["capabilities"] = capabilities
 
+        # Attach declared volumes to the app manifest (serve-time mounts)
+        if serve.volumes:
+            volumes_yaml = yaml_config.get("volumes", [])
+            for vol in serve.volumes:
+                volumes_yaml.append(
+                    {
+                        "name": vol.name,
+                        "mount": str(vol.serve_path),
+                    }
+                )
+            yaml_config["volumes"] = volumes_yaml
+
+        # If it has a php dependency, set the scaling mode to single_concurrency
+        has_php = any(dep.name == "php" for dep in serve.deps)
+        if has_php:
+            scaling = yaml_config.get("scaling", {})
+            scaling["mode"] = "single_concurrency"
+            yaml_config["scaling"] = scaling
+
         if "after_deploy" in serve.commands:
             jobs = yaml_config.get("jobs", [])
-            jobs.append({
-                "name": "after_deploy",
-                "trigger": "post-deployment",
-                "action": {
-                    "execute": {
-                        "command": "after_deploy"
-                    }
+            jobs.append(
+                {
+                    "name": "after_deploy",
+                    "trigger": "post-deployment",
+                    "action": {"execute": {"command": "after_deploy"}},
                 }
-            })
+            )
             yaml_config["jobs"] = jobs
 
         app_yaml = yaml.dump(yaml_config)
@@ -1043,12 +1077,6 @@ class WasmerBuilder:
             ],
         )
 
-    def serve_mount(self, name: str) -> str:
-        return self.inner_builder.serve_mount(name)
-
-    def get_asset(self, name: str) -> str:
-        return self.inner_builder.get_asset(name)
-
     def run_command(
         self, command: str, extra_args: Optional[List[str]] | None = None
     ) -> Any:
@@ -1101,6 +1129,7 @@ class Ctx:
         self.steps: List[Step] = []
         self.serves: Dict[str, Serve] = {}
         self.mounts: List[Mount] = []
+        self.volumes: List[Volume] = []
         self.services: Dict[str, Service] = {}
 
     def add_package(self, package: Package) -> str:
@@ -1123,6 +1152,8 @@ class Ctx:
             return self.steps[int(index[len("ref:step:") :])]
         elif index.startswith("ref:mount:"):
             return self.mounts[int(index[len("ref:mount:") :])]
+        elif index.startswith("ref:volume:"):
+            return self.volumes[int(index[len("ref:volume:") :])]
         elif index.startswith("ref:service:"):
             return self.services[index[len("ref:service:") :]]
         else:
@@ -1148,14 +1179,13 @@ class Ctx:
     def getenv(self, name: str) -> Optional[str]:
         return self.builder.getenv(name)
 
-    def get_asset(self, name: str) -> Optional[str]:
-        return self.builder.get_asset(name)
-
     def dep(self, name: str, version: Optional[str] = None) -> str:
         package = Package(name, version)
         return self.add_package(package)
-    
-    def service(self, name: str, provider: Literal["postgres", "mysql", "redis"]) -> str:
+
+    def service(
+        self, name: str, provider: Literal["postgres", "mysql", "redis"]
+    ) -> str:
         service = Service(name, provider)
         return self.add_service(service)
 
@@ -1167,10 +1197,10 @@ class Ctx:
         deps: List[str],
         commands: Dict[str, str],
         cwd: Optional[str] = None,
-        assets: Optional[Dict[str, str]] = None,
         prepare: Optional[List[str]] = None,
         workers: Optional[List[str]] = None,
         mounts: Optional[List[Mount]] = None,
+        volumes: Optional[List[Volume]] = None,
         env: Optional[Dict[str, str]] = None,
         services: Optional[List[str]] = None,
     ) -> str:
@@ -1188,7 +1218,6 @@ class Ctx:
             provider=provider,
             build=build_refs,
             cwd=cwd,
-            assets=assets,
             deps=dep_refs,
             commands=commands,
             prepare=prepare_steps,
@@ -1196,6 +1225,9 @@ class Ctx:
             mounts=self.get_refs([mount["ref"] for mount in mounts])
             if mounts
             else None,
+            volumes=self.get_refs([volume["ref"] for volume in volumes])
+            if volumes
+            else None,
             env=env,
             services=self.get_refs(services) if services else None,
         )
@@ -1219,14 +1251,15 @@ class Ctx:
         return self.add_step(step)
 
     def copy(
-        self, source: str, target: str, ignore: Optional[List[str]] = None
+        self,
+        source: str,
+        target: str,
+        ignore: Optional[List[str]] = None,
+        base: Optional[Literal["source", "assets"]] = None,
     ) -> Optional[str]:
-        step = CopyStep(source, target, ignore)
+        step = CopyStep(source, target, ignore, base or "source")
         return self.add_step(step)
 
-    def buildpath(self, name: str) -> str:
-        return str((self.builder.get_build_path() / name).absolute())
-
     def env(self, **env_vars: str) -> Optional[str]:
         step = EnvStep(env_vars)
         return self.add_step(step)
@@ -1246,8 +1279,18 @@ class Ctx:
             "serve": str(serve_path.absolute()),
         }
 
-    def serve_mount(self, name: str) -> Optional[str]:
-        return self.builder.serve_mount(name)
+    def add_volume(self, volume: Volume) -> Optional[str]:
+        self.volumes.append(volume)
+        return f"ref:volume:{len(self.volumes) - 1}"
+
+    def volume(self, name: str, serve: str) -> Optional[str]:
+        volume = Volume(name=name, serve_path=Path(serve))
+        ref = self.add_volume(volume)
+        return {
+            "ref": ref,
+            "name": name,
+            "serve": str(volume.serve_path),
+        }
 
 
 def print_help() -> None:
@@ -1260,6 +1303,13 @@ def print_help() -> None:
     console.print(panel)
 
 
+def download_file(url: str, path: Path) -> None:
+    response = requests.get(url)
+    response.raise_for_status()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(response.content)
+
+
 @app.command(name="auto")
 def auto(
     path: Path = typer.Argument(
@@ -1319,12 +1369,35 @@ def auto(
         None,
         help="Name of the Wasmer app.",
     ),
+    use_procfile: bool = typer.Option(
+        True,
+        help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
+    ),
+    install_command: Optional[str] = typer.Option(
+        None,
+        help="The install command to use (overwrites the default)",
+    ),
+    build_command: Optional[str] = typer.Option(
+        None,
+        help="The build command to use (overwrites the default)",
+    ),
+    start_command: Optional[str] = typer.Option(
+        None,
+        help="The start command to use (overwrites the default)",
+    ),
 ):
     if not path.exists():
         raise Exception(f"The path {path} does not exist")
 
     if not (path / "Shipit").exists() or regenerate or regenerate_path is not None:
-        generate(path, out=regenerate_path)
+        generate(
+            path,
+            out=regenerate_path,
+            use_procfile=use_procfile,
+            install_command=install_command,
+            build_command=build_command,
+            start_command=start_command,
+        )
 
     build(
         path,
@@ -1364,13 +1437,53 @@ def generate(
         None,
         help="Output path (defaults to the Shipit file in the provided path).",
     ),
+    use_procfile: bool = typer.Option(
+        True,
+        help="Use the Procfile to generate the default custom commands (install, build, start, after_deploy).",
+    ),
+    install_command: Optional[str] = typer.Option(
+        None,
+        help="The install command to use (overwrites the default)",
+    ),
+    build_command: Optional[str] = typer.Option(
+        None,
+        help="The build command to use (overwrites the default)",
+    ),
+    start_command: Optional[str] = typer.Option(
+        None,
+        help="The start command to use (overwrites the default)",
+    ),
 ):
     if not path.exists():
         raise Exception(f"The path {path} does not exist")
 
     if out is None:
         out = path / "Shipit"
-    content = generate_shipit(path)
+    custom_commands = CustomCommands()
+    # if (path / "Dockerfile").exists():
+    #     # We get the start command from the Dockerfile
+    #     with open(path / "Dockerfile", "r") as f:
+    #         cmd = None
+    #         for line in f:
+    #             if line.startswith("CMD "):
+    #                 cmd = line[4:].strip()
+    #                 cmd = json.loads(cmd)
+    #         # We get the last command
+    #         if cmd:
+    #             if isinstance(cmd, list):
+    #                 cmd = " ".join(cmd)
+    #             custom_commands.start = cmd
+    if use_procfile:
+        if (path / "Procfile").exists():
+            procfile = Procfile.loads((path / "Procfile").read_text())
+            custom_commands.start = procfile.get_start_command()
+    if start_command:
+        custom_commands.start = start_command
+    if install_command:
+        custom_commands.install = install_command
+    if build_command:
+        custom_commands.build = build_command
+    content = generate_shipit(path, custom_commands)
     out.write_text(content)
     console.print(f"[bold]Generated Shipit[/bold] at {out.absolute()}")
 
@@ -1529,15 +1642,12 @@ def build(
     mod.add_callable("serve", ctx.serve)
     mod.add_callable("run", ctx.run)
     mod.add_callable("mount", ctx.mount)
+    mod.add_callable("volume", ctx.volume)
     mod.add_callable("workdir", ctx.workdir)
     mod.add_callable("copy", ctx.copy)
     mod.add_callable("path", ctx.path)
-    mod.add_callable("buildpath", ctx.buildpath)
-    mod.add_callable("get_asset", ctx.get_asset)
     mod.add_callable("env", ctx.env)
     mod.add_callable("use", ctx.use)
-    # REMOVE ME
-    mod.add_callable("serve_mount", ctx.serve_mount)
 
     dialect = sl.Dialect.extended()
     dialect.enable_f_strings = True
@@ -1545,9 +1655,7 @@ def build(
     ast = sl.parse("shipit", source, dialect=dialect)
 
     sl.eval(mod, ast, glb)
-    # assert len(ctx.builds) == 1, "Only one build is allowed for now"
     assert len(ctx.serves) <= 1, "Only one serve is allowed for now"
-    # build = ctx.builds[0]
     env = {
         "PATH": "",
         "COLORTERM": os.environ.get("COLORTERM", ""),
@@ -1561,8 +1669,6 @@ def build(
     builder.build(env, serve.mounts, serve.build)
     if serve.prepare:
         builder.build_prepare(serve)
-    if serve.assets:
-        builder.build_assets(serve.assets)
     builder.build_serve(serve)
     builder.finalize_build(serve)
     if serve.prepare and not skip_prepare:
@@ -1580,7 +1686,7 @@ def main() -> None:
         app()
     except Exception as e:
         console.print(f"[bold red]{type(e).__name__}[/bold red]: {e}")
-        # raise e
+        raise e
 
 
 if __name__ == "__main__":

shipit/env.py

@@ -0,0 +1,30 @@
+import shlex
+import re
+
+def parse(content):
+    """
+    Parse the content of a .env file (a line-delimited KEY=value format) into a
+    dictionary mapping keys to values.
+    """
+    values = {}
+    for line in content.splitlines():
+        lexer = shlex.shlex(line, posix=True)
+        tokens = list(lexer)
+
+        # parses the assignment statement
+        if len(tokens) < 3:
+            continue
+
+        name, op = tokens[:2]
+        value = ''.join(tokens[2:])
+
+        if op != '=':
+            continue
+        if not re.match(r'[A-Za-z_][A-Za-z_0-9]*', name):
+            continue
+
+        value = value.replace(r'\n', '\n')
+        value = value.replace(r'\t', '\t')
+        values[name] = value
+
+    return values