genesis-devtools 0.0.4__py3-none-any.whl

genesis_devtools/builder/__init__.py

@@ -0,0 +1,17 @@
+#    Copyright 2025 Genesis Corporation.
+#
+#    All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from genesis_devtools.builder import dependency

genesis_devtools/builder/base.py

@@ -0,0 +1,196 @@
+#    Copyright 2025 Genesis Corporation.
+#
+#    All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import annotations
+
+import abc
+import os
+import typing as tp
+
+from genesis_devtools import constants as c
+
+
+class Image(tp.NamedTuple):
+    """Image representation."""
+
+    script: str
+    profile: c.ImageProfileType = "ubuntu_24"
+    format: c.ImageFormatType = "raw"
+    name: str | None = None
+    envs: tp.List[str] | None = None
+    override: tp.Dict[str, tp.Any] | None = None
+
+    @classmethod
+    def from_config(
+        cls, image_config: tp.Dict[str, tp.Any], work_dir: str
+    ) -> "Image":
+        """Create an image from configuration."""
+        script = image_config.pop("script")
+        if not os.path.isabs(script):
+            script = os.path.join(work_dir, script)
+        return cls(script=script, **image_config)
+
+
+class Element(tp.NamedTuple):
+    """Element representation."""
+
+    manifest: tp.Optional[str] = None
+    images: tp.Optional[tp.List[Image]] = None
+    artifacts: tp.Optional[tp.List[str]] = None
+
+    def __str__(self):
+        if self.manifest:
+            # TODO: Add implementation where manifest is used.
+            return "<Element manifest=...>"
+
+        if self.images and len(self.images) > 0:
+            name = ", ".join([f"{i.profile}" for i in self.images])
+            return f"<Element images={name}>"
+
+        return f"<Element {str(self)}>"
+
+    @classmethod
+    def from_config(
+        cls, element_config: tp.Dict[str, tp.Any], work_dir: str
+    ) -> "Element":
+        """Create an element from configuration."""
+        image_configs = element_config.pop("images", [])
+        images = [Image.from_config(img, work_dir) for img in image_configs]
+        return cls(images=images, **element_config)
+
+
+class AbstractDependency(abc.ABC):
+    """Abstract dependency item.
+
+    This class defines the interface for a dependency item.
+    """
+
+    dependencies_store: tp.List["AbstractDependency"] = []
+
+    def __init_subclass__(cls, **kwargs) -> None:
+        super().__init_subclass__()
+        cls.dependencies_store.append(cls)
+
+    @abc.abstractproperty
+    def img_dest(self) -> tp.Optional[str]:
+        """Destination for the image."""
+
+    @property
+    def local_path(self) -> tp.Optional[str]:
+        """Local path to the dependency."""
+        return None
+
+    @abc.abstractmethod
+    def fetch(self, output_dir: str) -> None:
+        """Fetch the dependency."""
+
+    @abc.abstractclassmethod
+    def from_config(
+        cls, dep_config: tp.Dict[str, tp.Any], work_dir: str
+    ) -> "AbstractDependency":
+        """Create a dependency item from configuration."""
+
+    @classmethod
+    def find_dependency(
+        cls, dep_config: tp.Dict[str, tp.Any], work_dir: str
+    ) -> tp.Optional["AbstractDependency"]:
+        """Probe all dependencies to find the right one."""
+        for dep in cls.dependencies_store:
+            try:
+                return dep.from_config(dep_config, work_dir)
+            except Exception:
+                pass
+
+        return None
+
+
+class AbstractImageBuilder(abc.ABC):
+    """Abstract image builder.
+
+    This class defines the interface for building images.
+    """
+
+    @abc.abstractmethod
+    def pre_build(
+        self,
+        image_dir: str,
+        image: Image,
+        deps: tp.List[AbstractDependency],
+        developer_keys: tp.Optional[str] = None,
+    ) -> None:
+        """Actions to prepare the environment for building the image."""
+
+    @abc.abstractmethod
+    def build(
+        self,
+        image_dir: str,
+        image: Image,
+        developer_keys: tp.Optional[str] = None,
+    ) -> None:
+        """Actions to build the image."""
+
+    @abc.abstractmethod
+    def post_build(
+        self,
+        image_dir: str,
+        image: Image,
+    ) -> None:
+        """Actions to perform after building the image."""
+
+    def run(
+        self,
+        image_dir: str,
+        image: Image,
+        deps: tp.List[AbstractDependency],
+        developer_keys: tp.Optional[str] = None,
+    ) -> None:
+        """Run the image builder."""
+        self.pre_build(image_dir, image, deps, developer_keys)
+        self.build(image_dir, image, developer_keys)
+        self.post_build(image_dir, image)
+
+
+class DummyImageBuilder(AbstractImageBuilder):
+    """Dummy image builder.
+
+    Dummy builder that does nothing.
+    """
+
+    def pre_build(
+        self,
+        image_dir: str,
+        image: Image,
+        deps: tp.List[AbstractDependency],
+        developer_keys: tp.Optional[str] = None,
+    ) -> None:
+        """Actions to prepare the environment for building the image."""
+        return None
+
+    def build(
+        self,
+        image_dir: str,
+        image: Image,
+        developer_keys: tp.Optional[str] = None,
+    ) -> None:
+        """Actions to build the image."""
+        return None
+
+    def post_build(
+        self,
+        image_dir: str,
+        image: Image,
+    ) -> None:
+        """Actions to perform after building the image."""
+        return None

genesis_devtools/builder/builder.py

@@ -0,0 +1,108 @@
+#    Copyright 2025 Genesis Corporation.
+#
+#    All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import typing as tp
+import tempfile
+
+from genesis_devtools.builder import base
+from genesis_devtools.logger import AbstractLogger, DummyLogger
+
+
+class SimpleBuilder:
+    """Abstract element builder.
+
+    This class defines the interface for building elements.
+    """
+
+    DEP_KEY = "deps"
+    ELEMENT_KEY = "elements"
+
+    def __init__(
+        self,
+        work_dir: str,
+        deps: tp.List[base.AbstractDependency],
+        elements: tp.List[base.Element],
+        image_builder: base.AbstractImageBuilder,
+        logger: tp.Optional[AbstractLogger] = None,
+    ) -> None:
+        super().__init__()
+        self._deps = deps
+        self._elements = elements
+        self._image_builder = image_builder
+        self._work_dir = work_dir
+        self._logger = logger or DummyLogger()
+
+    def fetch_dependency(self, deps_dir: str) -> None:
+        """Fetch common dependencies for elements."""
+        self._logger.important("Fetching dependencies")
+        for dep in self._deps:
+            self._logger.info(f"Fetching dependency: {dep}")
+            dep.fetch(deps_dir)
+
+    def build(
+        self,
+        build_dir: tp.Optional[str] = None,
+        developer_keys: tp.Optional[str] = None,
+    ) -> None:
+        """Build all elements."""
+        self._logger.important("Building elements")
+        for e in self._elements:
+            self._logger.info(f"Building element: {e}")
+            for img in e.images:
+                # The build_dir is used only for debugging purposes to observe
+                # the content of the image. In production, the image is built
+                # in a temporary directory.
+                if build_dir is not None:
+                    self._image_builder.run(
+                        build_dir, img, self._deps, developer_keys
+                    )
+                else:
+                    with tempfile.TemporaryDirectory() as temp_dir:
+                        self._image_builder.run(
+                            temp_dir, img, self._deps, developer_keys
+                        )
+
+    @classmethod
+    def from_config(
+        cls,
+        work_dir: str,
+        build_config: tp.Dict[str, tp.Any],
+        image_builder: base.AbstractImageBuilder,
+        logger: tp.Optional[AbstractLogger] = None,
+    ) -> "SimpleBuilder":
+        """Create a builder from configuration."""
+        # Prepare dependencies entries but do not fetch them
+        deps = []
+        dep_configs = build_config.get(cls.DEP_KEY, [])
+        for dep in dep_configs:
+            dep_item = base.AbstractDependency.find_dependency(dep, work_dir)
+            if dep_item is None:
+                raise ValueError(
+                    f"Unable to handle dependency: {dep}. Unknown type."
+                )
+            deps.append(dep_item)
+
+        # Prepare elements
+        element_configs = build_config.get(cls.ELEMENT_KEY, [])
+        elements = [
+            base.Element.from_config(elem, work_dir)
+            for elem in element_configs
+        ]
+
+        if not elements:
+            raise ValueError("No elements found in configuration")
+
+        return cls(work_dir, deps, elements, image_builder, logger)

genesis_devtools/builder/dependency.py

@@ -0,0 +1,125 @@
+#    Copyright 2025 Genesis Corporation.
+#
+#    All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import annotations
+
+import os
+import shutil
+import typing as tp
+
+import bazooka
+
+from genesis_devtools.builder import base
+
+
+class LocalPathDependency(base.AbstractDependency):
+    """Local path dependency item."""
+
+    def __init__(self, path: str, img_dest: str) -> None:
+        super().__init__()
+        self._path = path
+        self._img_dest = img_dest
+        self._local_path = None
+
+    @property
+    def img_dest(self) -> str | None:
+        """Destination for the image."""
+        return self._img_dest
+
+    @property
+    def local_path(self) -> str | None:
+        """Local path to the dependency."""
+        return self._local_path
+
+    def fetch(self, output_dir: str) -> None:
+        """Fetch the dependency."""
+        path = self._path
+        if os.path.isdir(path):
+            name = os.path.basename(path)
+            shutil.copytree(path, os.path.join(output_dir, name))
+            self._local_path = os.path.join(output_dir, name)
+        else:
+            shutil.copy(path, output_dir)
+            self._local_path = os.path.join(output_dir, os.path.basename(path))
+
+    def __str__(self):
+        return f"Local path -> {self._path}"
+
+    @classmethod
+    def from_config(
+        cls, dep_config: tp.Dict[str, tp.Any], work_dir: str
+    ) -> "LocalPathDependency":
+        """Create a dependency item from configuration."""
+        if "path" not in dep_config or "src" not in dep_config["path"]:
+            raise ValueError("Path not found in dependency configuration")
+
+        path = dep_config["path"]["src"]
+        img_dest = dep_config["dst"]
+
+        if not os.path.isabs(path):
+            path = os.path.join(work_dir, path)
+
+        return cls(path, img_dest)
+
+
+class HttpDependency(base.AbstractDependency):
+    """HTTP dependency item."""
+
+    CHUNK_SIZE = 8192
+
+    def __init__(self, endpoint: str, img_dest: str) -> None:
+        super().__init__()
+        self._endpoint = endpoint
+        self._img_dest = img_dest
+        self._local_path = None
+
+    @property
+    def img_dest(self) -> str | None:
+        """Destination for the image."""
+        return self._img_dest
+
+    @property
+    def local_path(self) -> str | None:
+        """Local path to the dependency."""
+        return self._local_path
+
+    def fetch(self, output_dir: str) -> None:
+        """Fetch the dependency."""
+        filename = os.path.basename(self._endpoint)
+        output_path = os.path.join(output_dir, filename)
+
+        with bazooka.get(self._endpoint, stream=True) as r:
+            r.raise_for_status()
+            with open(output_path, "wb") as f:
+                for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
+                    f.write(chunk)
+
+        self._local_path = output_path
+
+    def __str__(self):
+        return f"URL -> {self._endpoint}"
+
+    @classmethod
+    def from_config(
+        cls, dep_config: tp.Dict[str, tp.Any], work_dir: str
+    ) -> "LocalPathDependency":
+        """Create a dependency item from configuration."""
+        if "http" not in dep_config or "src" not in dep_config["http"]:
+            raise ValueError("URL not found in dependency configuration")
+
+        endpoint = dep_config["http"]["src"]
+        img_dest = dep_config["dst"]
+
+        return cls(endpoint, img_dest)

genesis_devtools/builder/packer.py

@@ -0,0 +1,201 @@
+#    Copyright 2025 Genesis Corporation.
+#
+#    All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import annotations
+
+import os
+import subprocess
+import typing as tp
+import shutil
+from importlib.resources import files
+
+from genesis_devtools.builder import base
+from genesis_devtools.logger import AbstractLogger, DummyLogger
+from genesis_devtools import constants as c
+
+
+file_provisioner_tmpl = """
+  provisioner "file" {{
+      source      = "{source}"
+      destination = "{tmp_destination}"
+  }}
+  provisioner "shell" {{
+    inline = [
+      "sudo mv {tmp_destination} {destination}",
+    ]
+  }}
+"""
+
+
+dev_keys_provisioner_tmpl = """
+  provisioner "file" {{
+      source      = "{source}"
+      destination = "/tmp/__dev_keys"
+  }}
+"""
+
+
+packer_build_tmpl = """
+variable "output_directory" {{
+  type    = string
+  default = "{output_directory}"
+}}
+
+build {{
+  source "qemu.{profile}" {{
+    name = "{name}"
+  }}
+
+  {file_provisioners}
+
+  provisioner "shell" {{
+    execute_command = "sudo -S env {{{{ .Vars }}}} {{{{ .Path }}}}"
+    script          = "{script}"
+    env             = {{
+      EXAMPLE_VARIABLE = "example_value"
+    }}
+  }}
+
+  {developer_keys}
+}}
+"""
+
+
+class PackerVariable(tp.NamedTuple):
+    name: str
+    value: str | int | float | list | dict | None = ""
+    var_tmpl: str = "{name} = {value}"
+
+    def render(self) -> str:
+        data = self._asdict()
+        data.pop("var_tmpl", None)
+
+        # Need quotes for strings in HCL
+        if isinstance(self.value, str):
+            data["value"] = f'"{self.value}"'
+
+        return self.var_tmpl.format(**data)
+
+    @classmethod
+    def variable_file_content(cls, overrides: tp.Dict[str, tp.Any]) -> str:
+        if not overrides:
+            return ""
+
+        variables = []
+        for k, v in overrides.items():
+            variables.append(cls(name=k, value=v))
+
+        return "\n".join([v.render() for v in variables])
+
+
+def _get_profile_files(base: str) -> str:
+    """Get base files for the image."""
+    profile_files = []
+    for bfile in files(f"{c.PKG_NAME}.packer.{base}").iterdir():
+        profile_files.append(bfile)
+
+    return profile_files
+
+
+class PackerBuilder(base.DummyImageBuilder):
+    """Packer image builder.
+
+    Packer image builder that uses Packer tools to build images.
+    """
+
+    def __init__(
+        self, output_dir: str, logger: AbstractLogger | None = None
+    ) -> None:
+        super().__init__()
+        self._logger = logger or DummyLogger()
+        self._output_dir = output_dir
+
+    def pre_build(
+        self,
+        image_dir: str,
+        image: base.Image,
+        deps: tp.List[base.AbstractDependency],
+        developer_keys: str | None = None,
+    ) -> None:
+        """Actions to prepare the environment for building the image."""
+
+        # Prepare the packer build file
+        # Data provisioners
+        provisioners = []
+        for i, dep in enumerate(deps):
+            tmp_dest = os.path.join(
+                "/tmp/", os.path.basename(dep.img_dest) + f"_{i}"
+            )
+            provisioners.append(
+                file_provisioner_tmpl.format(
+                    source=dep.local_path,
+                    destination=dep.img_dest,
+                    tmp_destination=tmp_dest,
+                )
+            )
+
+        # Prepare developer
+        if developer_keys:
+            dev_key_path = os.path.join(image_dir, "__dev_keys")
+            with open(dev_key_path, "w") as f:
+                f.write(developer_keys)
+            developer_keys_prov = dev_keys_provisioner_tmpl.format(
+                source=dev_key_path
+            )
+        else:
+            developer_keys_prov = ""
+
+        profile = image.profile.replace("_", "-")
+        packer_build = packer_build_tmpl.format(
+            profile=profile,
+            name=image.name or profile,
+            file_provisioners="\n".join(provisioners),
+            script=image.script,
+            developer_keys=developer_keys_prov,
+            output_directory=self._output_dir,
+        )
+
+        # Write the packer build file
+        main_path = os.path.join(image_dir, "main.pkr.hcl")
+        with open(main_path, "w") as f:
+            f.write(packer_build)
+
+        # Copy profile files to the image directory
+        profile_files = _get_profile_files(image.profile)
+        for bfile in profile_files:
+            shutil.copy(bfile, image_dir)
+
+        # Override variables if they are provided
+        if variables := PackerVariable.variable_file_content(
+            image.override or ()
+        ):
+            with open(
+                os.path.join(image_dir, "overrides.auto.pkrvars.hcl"), "w"
+            ) as f:
+                f.write(variables)
+
+        subprocess.run(["packer", "init", image_dir], check=True)
+
+    def build(
+        self,
+        image_dir: str,
+        image: base.Image,
+        developer_keys: str | None = None,
+    ) -> None:
+        """Actions to build the image."""
+        self._logger.important(f"Build image: {image.name}")
+        subprocess.run(
+            ["packer", "build", "-parallel-builds=1", image_dir], check=True
+        )